From c9fbad860458b4ca27265d2f84374cc488fb39dc Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Fri, 4 Jul 2025 16:00:33 +0300 Subject: [PATCH 1/4] add folder --- app/backup.py | 42 ++++++++++++++++++++++++++-------- app/config.py | 3 +++ app/config/config-template.ini | 5 ++++ app/repo.py | 6 ++++- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/app/backup.py b/app/backup.py index a2758ec..0e4d6d5 100644 --- a/app/backup.py +++ b/app/backup.py @@ -6,6 +6,26 @@ import re from repo import add_file_and_commit +def get_bcp_file_path(name: str) -> [str, str|None]: + if cfg.bcp.in_folder: + match = re.match(cfg.bcp.folder_name_pattern, name) + if match: + result = match.group(1) + bcp_file_path = os.path.join(cfg.bcp.dir, result, name) + if not os.path.exists(os.path.join(cfg.bcp.dir, result)): + os.makedirs(os.path.join(cfg.bcp.dir, result)) + log.info("create folder %r", result) + else: + log.info("folder %r exists", result) + return bcp_file_path, result + else: + return os.path.join(cfg.bcp.dir, name), None + else: + return os.path.join(cfg.bcp.dir, name), None + + + + def read_device_list() -> dict[str, dict[str, str]]: bcp_dev_file = os.path.join( @@ -106,11 +126,11 @@ def save_mikrotik_bcp(host: str, name: str) -> None: log.warning("Timeout read config from %r", name) return result = "\n".join(result.split("\n")[1:]) - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) def save_snr_bcp(host: str, name: str) -> None: @@ -129,11 +149,11 @@ def save_snr_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) def save_cisco_sb_bcp(host: str, name: str) -> None: @@ -152,10 +172,11 @@ def save_cisco_sb_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) def save_cisco_bcp(host: str, name: str) -> None: @@ -174,7 +195,8 @@ def save_cisco_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) diff --git a/app/config.py b/app/config.py index 0c4ae27..1c4c7b6 100644 --- a/app/config.py +++ b/app/config.py @@ -59,6 +59,9 @@ class ConfigBcp: file: str = config["bcp"].get("file") pattern: str = config["bcp"].get("pattern") start_at: str = config["bcp"].get("start_at") + in_folder: bool = config["bcp"].getboolean("in_folder") + if in_folder: + folder_name_pattern: str = config["bcp"].get("folder_name_pattern") @dataclass diff --git a/app/config/config-template.ini b/app/config/config-template.ini index 78a9498..7f649dd 100644 --- a/app/config/config-template.ini +++ b/app/config/config-template.ini @@ -35,6 +35,11 @@ file: dev_list.txt pattern:(?P\S+) (?P\d+\.\d+\.\d+\.\d+) (?P\S+) (?P\S+) # Во сколько будет делаться бэкап start_at: 18:59 +# True \ False - Создание папок внутри репозитория. Если False, то все файлы будут сохраняться в корень +in_folder: True +# Регулярное выражение по которому будет парситься имя папки из названия девайса +folder_name_pattern: ^(.*?)- + [net_dev] debug: False diff --git a/app/repo.py b/app/repo.py index 1964383..0ae1136 100644 --- a/app/repo.py +++ b/app/repo.py @@ -52,9 +52,11 @@ def check_bcp_repo(path: str = cfg.bcp.dir) -> None: log.warning("Error: %r", e) -def add_file_and_commit(file_name: str) -> None: +def add_file_and_commit(file_name: str, file_folder: str) -> None: current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") repo = Repo(cfg.bcp.dir) + if file_folder is not None: + file_name = file_folder + "/" + file_name if file_name in repo.untracked_files: repo.git.add(file_name) repo.index.commit("add %s %s" % (file_name, current_time)) @@ -68,6 +70,8 @@ def add_file_and_commit(file_name: str) -> None: log.info("no changed in file %r", file_name) + + def push_to_remote(): repo = Repo(cfg.bcp.dir) remote = repo.remote(name=cfg.git.branch) From e1a28abc61c70baccc1aa48c43f0497ee596845f Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Fri, 4 Jul 2025 16:56:55 +0300 Subject: [PATCH 2/4] add remove local backups --- app/backup.py | 3 +++ app/config.py | 1 + app/config/config-template.ini | 2 ++ app/main.py | 4 +++- app/repo.py | 30 ++++++++++++++++++++++++++++-- 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/app/backup.py b/app/backup.py index 0e4d6d5..7683bc3 100644 --- a/app/backup.py +++ b/app/backup.py @@ -137,6 +137,7 @@ def save_snr_bcp(host: str, name: str) -> None: connection = connect_to_device( vendor="snr", host=host, + name=name, ) if connection is None: return @@ -160,6 +161,7 @@ def save_cisco_sb_bcp(host: str, name: str) -> None: connection = connect_to_device( vendor="cisco_sb", host=host, + name=name, ) if connection is None: return @@ -183,6 +185,7 @@ def save_cisco_bcp(host: str, name: str) -> None: connection = connect_to_device( vendor="cisco", host=host, + name=name, ) if connection is None: return diff --git a/app/config.py b/app/config.py index 1c4c7b6..28e234b 100644 --- a/app/config.py +++ b/app/config.py @@ -84,6 +84,7 @@ class ConfigGit: protocol: str = config["git"].get("protocol") username: str = os.getenv("GIT_USERNAME") token: str = os.getenv("GIT_TOKEN") + remove_local: bool = config["git"].getboolean("remove_local") @dataclass class ConfigMail: diff --git a/app/config/config-template.ini b/app/config/config-template.ini index 7f649dd..2680231 100644 --- a/app/config/config-template.ini +++ b/app/config/config-template.ini @@ -27,6 +27,8 @@ protocol: https remote: gitlab.example.com.ru/secure/backup-network-device.git # ветка branch: main +# Удаление локального репозитория после push +remove_local: False [bcp] # Файл со списком устройств. Должен лежать в папке config diff --git a/app/main.py b/app/main.py index 3ddb90d..fa203ee 100644 --- a/app/main.py +++ b/app/main.py @@ -8,7 +8,7 @@ from backup import ( save_cisco_bcp, ) from messages import send_mail -from repo import check_bcp_repo, push_to_remote +from repo import check_bcp_repo, push_to_remote, delete_all_files import schedule import time import os @@ -48,6 +48,8 @@ def main(): save_cisco_bcp(host=current_device["ip"], name=current_device["name"]) if cfg.git.push: push_to_remote() + if cfg.git.remove_local: + delete_all_files() if cfg.mail.send: if os.path.isfile(log_file): with open(log_file, 'r', encoding='utf-8') as file: diff --git a/app/repo.py b/app/repo.py index 0ae1136..86e4c1d 100644 --- a/app/repo.py +++ b/app/repo.py @@ -1,8 +1,10 @@ -from git import Repo, InvalidGitRepositoryError, Git +from git import Repo, InvalidGitRepositoryError, Git, rmtree from config import cfg import logging as log from datetime import datetime - +import os +import shutil +import stat def check_remote_repo(repo: Repo): log.info("check remote repo ") @@ -77,3 +79,27 @@ def push_to_remote(): remote = repo.remote(name=cfg.git.branch) remote.push(refspec=f"{cfg.git.branch}:{cfg.git.branch}") log.info("push to remote repo %r", cfg.git.branch) + + +def delete_all_files(path: str = cfg.bcp.dir) -> None: + + if os.path.exists(path) and os.path.isdir(path): + for filename in os.listdir(path): + file_path = os.path.join(path, filename) + if filename == ".git": + log.info('start remove read only for .git') + for root, dirs, files in os.walk(file_path): + os.chmod(root, stat.S_IWRITE) + for file in files: + d_file_path = os.path.join(root, file) + os.chmod(d_file_path, stat.S_IWRITE) + log.info('end remove read only for .git') + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.remove(file_path) + log.info("Successfully deleting: %s", file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + log.info("Successfully deleting: %s", file_path) + except Exception as e: + log.warning("Error deleting %s: %s", file_path, e) \ No newline at end of file From 5755e102125865df741c8680ff42500e73383116 Mon Sep 17 00:00:00 2001 From: "s.mostryukov" Date: Wed, 9 Jul 2025 09:27:14 +0300 Subject: [PATCH 3/4] add minimum string count --- app/backup.py | 14 ++++++++++++++ app/config.py | 1 + app/config/config-template.ini | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/backup.py b/app/backup.py index 7683bc3..abb544e 100644 --- a/app/backup.py +++ b/app/backup.py @@ -104,6 +104,8 @@ def send_command(connection: ConnectHandler, command: str) -> str: result = "NetmikoTimeoutException" except exceptions.ReadTimeout: result = "ReadTimeout" + if len(result.split("\n")) < cfg.bcp.min_string_count: + result = 'NotEnoughLines' return result @@ -125,6 +127,9 @@ def save_mikrotik_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return result = "\n".join(result.split("\n")[1:]) bcp_file_name, file_folder = get_bcp_file_path(name) with open(bcp_file_name, "w") as f: @@ -150,6 +155,9 @@ def save_snr_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return bcp_file_name, file_folder = get_bcp_file_path(name) with open(bcp_file_name, "w") as f: f.write(result) @@ -174,6 +182,9 @@ def save_cisco_sb_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return bcp_file_name, file_folder = get_bcp_file_path(name) with open(bcp_file_name, "w") as f: f.write(result) @@ -198,6 +209,9 @@ def save_cisco_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return bcp_file_name, file_folder = get_bcp_file_path(name) with open(bcp_file_name, "w") as f: f.write(result) diff --git a/app/config.py b/app/config.py index 28e234b..8dece60 100644 --- a/app/config.py +++ b/app/config.py @@ -62,6 +62,7 @@ class ConfigBcp: in_folder: bool = config["bcp"].getboolean("in_folder") if in_folder: folder_name_pattern: str = config["bcp"].get("folder_name_pattern") + min_string_count: int = config["bcp"].getint("min_string_count") @dataclass diff --git a/app/config/config-template.ini b/app/config/config-template.ini index 2680231..7b3434b 100644 --- a/app/config/config-template.ini +++ b/app/config/config-template.ini @@ -41,7 +41,8 @@ start_at: 18:59 in_folder: True # Регулярное выражение по которому будет парситься имя папки из названия девайса folder_name_pattern: ^(.*?)- - +# Если количество строк в конфиге меньше указанного значения - ошибка, изменения не записываются. +min_string_count: 50 [net_dev] debug: False From 084f993e278be1b2af7c7aaa923f6aea44a67b66 Mon Sep 17 00:00:00 2001 From: sergey Date: Wed, 9 Jul 2025 09:33:49 +0300 Subject: [PATCH 4/4] upd compose --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index e3fc334..7cb1c5d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3.3' services: net-backup: - image: git.sm8255082.ru/osnova/net-backup:1.1.6 + image: git.sm8255082.ru/osnova/net-backup:1.2.0 restart: "no" volumes: - ./logs:/app/logs