From e80821e6da305c4bd0b837f9074030f43b0c42fb Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Wed, 18 Jun 2025 16:38:54 +0300 Subject: [PATCH 1/8] docker-compose + exception fix --- app/backup.py | 2 ++ docker/docker-compose.yaml | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/backup.py b/app/backup.py index a4702f8..3724716 100644 --- a/app/backup.py +++ b/app/backup.py @@ -76,6 +76,8 @@ def send_command(connection: ConnectHandler, command: str) -> str: result: str = connection.send_command(command, read_timeout=30) except exceptions.NetmikoTimeoutException: result = "NetmikoTimeoutException" + except exceptions.ReadTimeout: + result = "NetmikoTimeoutException" return result diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 8c980b7..6939660 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,10 +1,8 @@ version: '3.3' - services: - net-backup: - image: git.sm8255082.ru/osnova/net-backup:1.0.0 - restart: always + image: git.sm8255082.ru/osnova/net-backup:1.1.0 + restart: "no" volumes: - ./logs:/app/logs - ./config:/app/config @@ -12,7 +10,9 @@ services: environment: - TZ=Europe/Moscow - NET_DEV_USER=admin - - NET_DEV_DOMAIN=local + #- NET_DEV_DOMAIN=local - NET_DEV_PWD=password - - GIT_USERNAME=git_user - - GIT_TOKEN=bla-bla-lba \ No newline at end of file + #- GIT_USERNAME=git_user + #- GIT_TOKEN=bla-bla-lba + #- MAIL_USER=mail_user + #- MAIL_PWD=bla-bla-lba \ No newline at end of file From 49e62892c89db28fdd447b94dfe37b17709c8abe Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Thu, 19 Jun 2025 13:56:32 +0300 Subject: [PATCH 2/8] read timeout in config --- app/backup.py | 2 +- app/config.py | 1 + app/config/config-template.ini | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/backup.py b/app/backup.py index 3724716..c341940 100644 --- a/app/backup.py +++ b/app/backup.py @@ -73,7 +73,7 @@ def connect_to_device(vendor: str, host: str, name: str="") -> ConnectHandler: def send_command(connection: ConnectHandler, command: str) -> str: try: - result: str = connection.send_command(command, read_timeout=30) + result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout) except exceptions.NetmikoTimeoutException: result = "NetmikoTimeoutException" except exceptions.ReadTimeout: diff --git a/app/config.py b/app/config.py index 32960c4..790987c 100644 --- a/app/config.py +++ b/app/config.py @@ -67,6 +67,7 @@ class ConfigNetDev: domain: str = os.getenv("NET_DEV_DOMAIN") pwd: str = os.getenv("NET_DEV_PWD") ssh_port: int = config["net_dev"].getint("ssh_port") + read_timeout: int = config["net_dev"].getint("read_timeout") @dataclass diff --git a/app/config/config-template.ini b/app/config/config-template.ini index 76ee350..7024047 100644 --- a/app/config/config-template.ini +++ b/app/config/config-template.ini @@ -39,6 +39,7 @@ start_at: 18:59 [net_dev] # Порт на который подключаться по ssh ssh_port: 22 +read_timeout: 30 [log] # Уровень логов в консоль From 1c3ee168537f2c66cbf8d50b66f2e827d42cc5c0 Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Thu, 19 Jun 2025 14:49:29 +0300 Subject: [PATCH 3/8] fix exception --- app/backup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/backup.py b/app/backup.py index c341940..2a5e546 100644 --- a/app/backup.py +++ b/app/backup.py @@ -77,7 +77,7 @@ def send_command(connection: ConnectHandler, command: str) -> str: except exceptions.NetmikoTimeoutException: result = "NetmikoTimeoutException" except exceptions.ReadTimeout: - result = "NetmikoTimeoutException" + result = "ReadTimeout" return result @@ -93,6 +93,9 @@ def save_mikrotik_bcp(host: str, name: str) -> None: result = send_command(connection, "export terse show-sensitive") connection.disconnect() if result == "NetmikoTimeoutException": + log.warning("Timeout error %r", name) + return + elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return log.info("disconnected from %r", name) From 744ce9f6d96b96736cad3aefe5f854516e39604a Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Fri, 20 Jun 2025 16:50:46 +0300 Subject: [PATCH 4/8] add debug connection --- app/backup.py | 9 ++++++++- app/config.py | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/backup.py b/app/backup.py index 2a5e546..cea50e1 100644 --- a/app/backup.py +++ b/app/backup.py @@ -37,12 +37,16 @@ def read_device_list() -> dict[str, dict[str, str]]: def connect_to_device(vendor: str, host: str, name: str="") -> ConnectHandler: + log_dir: str = os.path.join(os.path.abspath(os.path.dirname(__file__)), "logs") + conn_conf: dict = { "host": host, "username": cfg.net_dev.user, "password": cfg.net_dev.pwd, "port": cfg.net_dev.ssh_port, } + if cfg.net_dev.debug: + conn_conf["session_log"] = os.path.join(log_dir, 'session_log.log') if vendor.lower() == "mikrotik": conn_conf["device_type"] = "mikrotik_routeros" if cfg.net_dev.domain is not None: @@ -72,8 +76,10 @@ def connect_to_device(vendor: str, host: str, name: str="") -> ConnectHandler: def send_command(connection: ConnectHandler, command: str) -> str: + log.info("send command") try: result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout) + log.info('command send success') except exceptions.NetmikoTimeoutException: result = "NetmikoTimeoutException" except exceptions.ReadTimeout: @@ -92,13 +98,14 @@ def save_mikrotik_bcp(host: str, name: str) -> None: return result = send_command(connection, "export terse show-sensitive") connection.disconnect() + log.info("disconnected from %r", name) if result == "NetmikoTimeoutException": log.warning("Timeout error %r", name) return elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - log.info("disconnected from %r", name) + result = "\n".join(result.split("\n")[1:]) with open(os.path.join(cfg.bcp.dir, name), "w") as f: f.write(result) diff --git a/app/config.py b/app/config.py index 790987c..0c4ae27 100644 --- a/app/config.py +++ b/app/config.py @@ -68,6 +68,7 @@ class ConfigNetDev: pwd: str = os.getenv("NET_DEV_PWD") ssh_port: int = config["net_dev"].getint("ssh_port") read_timeout: int = config["net_dev"].getint("read_timeout") + debug: bool = config["net_dev"].getboolean("debug") @dataclass From fced69656ec77366ebd4dd4eb55e0cb06cf98898 Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Sun, 22 Jun 2025 13:25:56 +0300 Subject: [PATCH 5/8] add debug connection in template --- app/config/config-template.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/app/config/config-template.ini b/app/config/config-template.ini index 7024047..2dc4c3e 100644 --- a/app/config/config-template.ini +++ b/app/config/config-template.ini @@ -37,6 +37,7 @@ pattern:(?P\S+) (?P\d+\.\d+\.\d+\.\d+) (?P\S+) (?P\S+) start_at: 18:59 [net_dev] +debug: False # Порт на который подключаться по ssh ssh_port: 22 read_timeout: 30 From 86744e5d81d189e5a6817e7863aafbdde2e02650 Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Sun, 22 Jun 2025 14:11:55 +0300 Subject: [PATCH 6/8] add debug connection in template --- app/backup.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/app/backup.py b/app/backup.py index cea50e1..492f7aa 100644 --- a/app/backup.py +++ b/app/backup.py @@ -83,7 +83,11 @@ def send_command(connection: ConnectHandler, command: str) -> str: except exceptions.NetmikoTimeoutException: result = "NetmikoTimeoutException" except exceptions.ReadTimeout: - result = "ReadTimeout" + try: + result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout) + log.info('the second time command send success') + except exceptions.ReadTimeout: + result = "ReadTimeout" return result @@ -105,7 +109,6 @@ def save_mikrotik_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - result = "\n".join(result.split("\n")[1:]) with open(os.path.join(cfg.bcp.dir, name), "w") as f: f.write(result) @@ -125,7 +128,10 @@ def save_snr_bcp(host: str, name: str) -> None: connection.disconnect() log.info("disconnected from %r", name) if result == "NetmikoTimeoutException": - log.warning("Timeout read config from %s", name) + log.warning("Timeout error %r", name) + return + elif result == "ReadTimeout": + log.warning("Timeout read config from %r", name) return with open(os.path.join(cfg.bcp.dir, name), "w") as f: f.write(result) @@ -145,7 +151,10 @@ def save_cisco_sb_bcp(host: str, name: str) -> None: connection.disconnect() log.info("disconnected from %s", name) if result == "NetmikoTimeoutException": - log.warning("Timeout read config from %s", name) + log.warning("Timeout error %r", name) + return + elif result == "ReadTimeout": + log.warning("Timeout read config from %r", name) return with open(os.path.join(cfg.bcp.dir, name), "w") as f: f.write(result) @@ -164,6 +173,9 @@ def save_cisco_bcp(host: str, name: str) -> None: connection.disconnect() log.info("disconnected from %r", name) if result == "NetmikoTimeoutException": + log.warning("Timeout error %r", name) + return + elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return with open(os.path.join(cfg.bcp.dir, name), "w") as f: From df0e4da626c10534cdd969dd7beeb631ea4b2f51 Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Sun, 22 Jun 2025 14:23:00 +0300 Subject: [PATCH 7/8] add delay factor --- app/backup.py | 6 +----- app/config.py | 1 + app/config/config-template.ini | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/backup.py b/app/backup.py index 492f7aa..e5bcea8 100644 --- a/app/backup.py +++ b/app/backup.py @@ -78,15 +78,11 @@ def connect_to_device(vendor: str, host: str, name: str="") -> ConnectHandler: def send_command(connection: ConnectHandler, command: str) -> str: log.info("send command") try: - result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout) + result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout, delay_factor=2) log.info('command send success') except exceptions.NetmikoTimeoutException: result = "NetmikoTimeoutException" except exceptions.ReadTimeout: - try: - result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout) - log.info('the second time command send success') - except exceptions.ReadTimeout: result = "ReadTimeout" return result diff --git a/app/config.py b/app/config.py index 0c4ae27..4f8cd6b 100644 --- a/app/config.py +++ b/app/config.py @@ -68,6 +68,7 @@ class ConfigNetDev: pwd: str = os.getenv("NET_DEV_PWD") ssh_port: int = config["net_dev"].getint("ssh_port") read_timeout: int = config["net_dev"].getint("read_timeout") + delay_factor: int = config["net_dev"].getint("delay_factor") debug: bool = config["net_dev"].getboolean("debug") diff --git a/app/config/config-template.ini b/app/config/config-template.ini index 2dc4c3e..dc7cfb8 100644 --- a/app/config/config-template.ini +++ b/app/config/config-template.ini @@ -41,6 +41,7 @@ debug: False # Порт на который подключаться по ssh ssh_port: 22 read_timeout: 30 +delay_factor: 2 [log] # Уровень логов в консоль From 228019695620bbad5e718ae2b74a3c2366e639f1 Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Sun, 22 Jun 2025 14:26:32 +0300 Subject: [PATCH 8/8] fix delay factor --- app/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/backup.py b/app/backup.py index e5bcea8..21dce7b 100644 --- a/app/backup.py +++ b/app/backup.py @@ -65,7 +65,7 @@ def connect_to_device(vendor: str, host: str, name: str="") -> ConnectHandler: conn_conf["username"] = cfg.net_dev.domain + "\\" + cfg.net_dev.user try: connection: ConnectHandler = ConnectHandler(**conn_conf) - log.info("connect to %s %s", name, host) + log.info("connect to %s %s %s", vendor, name, host) return connection except exceptions.NetmikoAuthenticationException: log.warning("Authentication error %s %s ", name, host) @@ -78,7 +78,7 @@ def connect_to_device(vendor: str, host: str, name: str="") -> ConnectHandler: def send_command(connection: ConnectHandler, command: str) -> str: log.info("send command") try: - result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout, delay_factor=2) + result: str = connection.send_command(command, read_timeout=cfg.net_dev.read_timeout, delay_factor=cfg.net_dev.delay_factor) log.info('command send success') except exceptions.NetmikoTimeoutException: result = "NetmikoTimeoutException"