diff --git a/app/backup.py b/app/backup.py index a2758ec..abb544e 100644 --- a/app/backup.py +++ b/app/backup.py @@ -6,6 +6,26 @@ import re from repo import add_file_and_commit +def get_bcp_file_path(name: str) -> [str, str|None]: + if cfg.bcp.in_folder: + match = re.match(cfg.bcp.folder_name_pattern, name) + if match: + result = match.group(1) + bcp_file_path = os.path.join(cfg.bcp.dir, result, name) + if not os.path.exists(os.path.join(cfg.bcp.dir, result)): + os.makedirs(os.path.join(cfg.bcp.dir, result)) + log.info("create folder %r", result) + else: + log.info("folder %r exists", result) + return bcp_file_path, result + else: + return os.path.join(cfg.bcp.dir, name), None + else: + return os.path.join(cfg.bcp.dir, name), None + + + + def read_device_list() -> dict[str, dict[str, str]]: bcp_dev_file = os.path.join( @@ -84,6 +104,8 @@ def send_command(connection: ConnectHandler, command: str) -> str: result = "NetmikoTimeoutException" except exceptions.ReadTimeout: result = "ReadTimeout" + if len(result.split("\n")) < cfg.bcp.min_string_count: + result = 'NotEnoughLines' return result @@ -105,18 +127,22 @@ def save_mikrotik_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return result = "\n".join(result.split("\n")[1:]) - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) def save_snr_bcp(host: str, name: str) -> None: connection = connect_to_device( vendor="snr", host=host, + name=name, ) if connection is None: return @@ -129,17 +155,21 @@ def save_snr_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) def save_cisco_sb_bcp(host: str, name: str) -> None: connection = connect_to_device( vendor="cisco_sb", host=host, + name=name, ) if connection is None: return @@ -152,16 +182,21 @@ def save_cisco_sb_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) def save_cisco_bcp(host: str, name: str) -> None: connection = connect_to_device( vendor="cisco", host=host, + name=name, ) if connection is None: return @@ -174,7 +209,11 @@ def save_cisco_bcp(host: str, name: str) -> None: elif result == "ReadTimeout": log.warning("Timeout read config from %r", name) return - with open(os.path.join(cfg.bcp.dir, name), "w") as f: + elif result == "NotEnoughLines": + log.warning("Small number of lines %r", name) + return + bcp_file_name, file_folder = get_bcp_file_path(name) + with open(bcp_file_name, "w") as f: f.write(result) log.info("Backup saved") - add_file_and_commit(file_name=name) + add_file_and_commit(file_name=name, file_folder=file_folder) diff --git a/app/config.py b/app/config.py index 0c4ae27..8dece60 100644 --- a/app/config.py +++ b/app/config.py @@ -59,6 +59,10 @@ class ConfigBcp: file: str = config["bcp"].get("file") pattern: str = config["bcp"].get("pattern") start_at: str = config["bcp"].get("start_at") + in_folder: bool = config["bcp"].getboolean("in_folder") + if in_folder: + folder_name_pattern: str = config["bcp"].get("folder_name_pattern") + min_string_count: int = config["bcp"].getint("min_string_count") @dataclass @@ -81,6 +85,7 @@ class ConfigGit: protocol: str = config["git"].get("protocol") username: str = os.getenv("GIT_USERNAME") token: str = os.getenv("GIT_TOKEN") + remove_local: bool = config["git"].getboolean("remove_local") @dataclass class ConfigMail: diff --git a/app/config/config-template.ini b/app/config/config-template.ini index 78a9498..7b3434b 100644 --- a/app/config/config-template.ini +++ b/app/config/config-template.ini @@ -27,6 +27,8 @@ protocol: https remote: gitlab.example.com.ru/secure/backup-network-device.git # ветка branch: main +# Удаление локального репозитория после push +remove_local: False [bcp] # Файл со списком устройств. Должен лежать в папке config @@ -35,6 +37,12 @@ file: dev_list.txt pattern:(?P\S+) (?P\d+\.\d+\.\d+\.\d+) (?P\S+) (?P\S+) # Во сколько будет делаться бэкап start_at: 18:59 +# True \ False - Создание папок внутри репозитория. Если False, то все файлы будут сохраняться в корень +in_folder: True +# Регулярное выражение по которому будет парситься имя папки из названия девайса +folder_name_pattern: ^(.*?)- +# Если количество строк в конфиге меньше указанного значения - ошибка, изменения не записываются. +min_string_count: 50 [net_dev] debug: False diff --git a/app/main.py b/app/main.py index 3ddb90d..fa203ee 100644 --- a/app/main.py +++ b/app/main.py @@ -8,7 +8,7 @@ from backup import ( save_cisco_bcp, ) from messages import send_mail -from repo import check_bcp_repo, push_to_remote +from repo import check_bcp_repo, push_to_remote, delete_all_files import schedule import time import os @@ -48,6 +48,8 @@ def main(): save_cisco_bcp(host=current_device["ip"], name=current_device["name"]) if cfg.git.push: push_to_remote() + if cfg.git.remove_local: + delete_all_files() if cfg.mail.send: if os.path.isfile(log_file): with open(log_file, 'r', encoding='utf-8') as file: diff --git a/app/repo.py b/app/repo.py index 1964383..86e4c1d 100644 --- a/app/repo.py +++ b/app/repo.py @@ -1,8 +1,10 @@ -from git import Repo, InvalidGitRepositoryError, Git +from git import Repo, InvalidGitRepositoryError, Git, rmtree from config import cfg import logging as log from datetime import datetime - +import os +import shutil +import stat def check_remote_repo(repo: Repo): log.info("check remote repo ") @@ -52,9 +54,11 @@ def check_bcp_repo(path: str = cfg.bcp.dir) -> None: log.warning("Error: %r", e) -def add_file_and_commit(file_name: str) -> None: +def add_file_and_commit(file_name: str, file_folder: str) -> None: current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") repo = Repo(cfg.bcp.dir) + if file_folder is not None: + file_name = file_folder + "/" + file_name if file_name in repo.untracked_files: repo.git.add(file_name) repo.index.commit("add %s %s" % (file_name, current_time)) @@ -68,8 +72,34 @@ def add_file_and_commit(file_name: str) -> None: log.info("no changed in file %r", file_name) + + def push_to_remote(): repo = Repo(cfg.bcp.dir) remote = repo.remote(name=cfg.git.branch) remote.push(refspec=f"{cfg.git.branch}:{cfg.git.branch}") log.info("push to remote repo %r", cfg.git.branch) + + +def delete_all_files(path: str = cfg.bcp.dir) -> None: + + if os.path.exists(path) and os.path.isdir(path): + for filename in os.listdir(path): + file_path = os.path.join(path, filename) + if filename == ".git": + log.info('start remove read only for .git') + for root, dirs, files in os.walk(file_path): + os.chmod(root, stat.S_IWRITE) + for file in files: + d_file_path = os.path.join(root, file) + os.chmod(d_file_path, stat.S_IWRITE) + log.info('end remove read only for .git') + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.remove(file_path) + log.info("Successfully deleting: %s", file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + log.info("Successfully deleting: %s", file_path) + except Exception as e: + log.warning("Error deleting %s: %s", file_path, e) \ No newline at end of file diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index e3fc334..7cb1c5d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3.3' services: net-backup: - image: git.sm8255082.ru/osnova/net-backup:1.1.6 + image: git.sm8255082.ru/osnova/net-backup:1.2.0 restart: "no" volumes: - ./logs:/app/logs