Merge pull request 'dev1.2.0' (#8) from dev1.2 into main

Reviewed-on: #8

app/backup.py

@@ -6,6 +6,26 @@ import re
 from repo import add_file_and_commit
 
 
+def get_bcp_file_path(name: str) -> [str, str|None]:
+    if cfg.bcp.in_folder:
+        match = re.match(cfg.bcp.folder_name_pattern, name)
+        if match:
+            result = match.group(1)
+            bcp_file_path = os.path.join(cfg.bcp.dir, result, name)
+            if not os.path.exists(os.path.join(cfg.bcp.dir, result)):
+                os.makedirs(os.path.join(cfg.bcp.dir, result))
+                log.info("create folder %r", result)
+            else:
+                log.info("folder %r exists", result)
+            return bcp_file_path, result
+        else:
+            return os.path.join(cfg.bcp.dir, name), None
+    else:
+        return os.path.join(cfg.bcp.dir, name), None
+
+
+
+
 def read_device_list() -> dict[str, dict[str, str]]:
 
     bcp_dev_file = os.path.join(
@@ -84,6 +104,8 @@ def send_command(connection: ConnectHandler, command: str) -> str:
         result = "NetmikoTimeoutException"
     except exceptions.ReadTimeout:
             result = "ReadTimeout"
+    if len(result.split("\n")) < cfg.bcp.min_string_count:
+        result = 'NotEnoughLines'
     return result
 
 
@@ -105,18 +127,22 @@ def save_mikrotik_bcp(host: str, name: str) -> None:
     elif result == "ReadTimeout":
         log.warning("Timeout read config from %r", name)
         return
+    elif result == "NotEnoughLines":
+        log.warning("Small number of lines %r", name)
+        return
     result = "\n".join(result.split("\n")[1:])
-    with open(os.path.join(cfg.bcp.dir, name), "w") as f:
+    bcp_file_name, file_folder = get_bcp_file_path(name)
+    with open(bcp_file_name, "w") as f:
         f.write(result)
     log.info("Backup saved")
-
-    add_file_and_commit(file_name=name)
+    add_file_and_commit(file_name=name, file_folder=file_folder)
 
 
 def save_snr_bcp(host: str, name: str) -> None:
     connection = connect_to_device(
         vendor="snr",
         host=host,
+        name=name,
     )
     if connection is None:
         return
@@ -129,17 +155,21 @@ def save_snr_bcp(host: str, name: str) -> None:
     elif result == "ReadTimeout":
         log.warning("Timeout read config from %r", name)
         return
-    with open(os.path.join(cfg.bcp.dir, name), "w") as f:
+    elif result == "NotEnoughLines":
+        log.warning("Small number of lines %r", name)
+        return
+    bcp_file_name, file_folder = get_bcp_file_path(name)
+    with open(bcp_file_name, "w") as f:
         f.write(result)
     log.info("Backup saved")
-
-    add_file_and_commit(file_name=name)
+    add_file_and_commit(file_name=name, file_folder=file_folder)
 
 
 def save_cisco_sb_bcp(host: str, name: str) -> None:
     connection = connect_to_device(
         vendor="cisco_sb",
         host=host,
+        name=name,
     )
     if connection is None:
         return
@@ -152,16 +182,21 @@ def save_cisco_sb_bcp(host: str, name: str) -> None:
     elif result == "ReadTimeout":
         log.warning("Timeout read config from %r", name)
         return
-    with open(os.path.join(cfg.bcp.dir, name), "w") as f:
+    elif result == "NotEnoughLines":
+        log.warning("Small number of lines %r", name)
+        return
+    bcp_file_name, file_folder = get_bcp_file_path(name)
+    with open(bcp_file_name, "w") as f:
         f.write(result)
     log.info("Backup saved")
-    add_file_and_commit(file_name=name)
+    add_file_and_commit(file_name=name, file_folder=file_folder)
 
 
 def save_cisco_bcp(host: str, name: str) -> None:
     connection = connect_to_device(
         vendor="cisco",
         host=host,
+        name=name,
     )
     if connection is None:
         return
@@ -174,7 +209,11 @@ def save_cisco_bcp(host: str, name: str) -> None:
     elif result == "ReadTimeout":
         log.warning("Timeout read config from %r", name)
         return
-    with open(os.path.join(cfg.bcp.dir, name), "w") as f:
+    elif result == "NotEnoughLines":
+        log.warning("Small number of lines %r", name)
+        return
+    bcp_file_name, file_folder = get_bcp_file_path(name)
+    with open(bcp_file_name, "w") as f:
         f.write(result)
     log.info("Backup saved")
-    add_file_and_commit(file_name=name)
+    add_file_and_commit(file_name=name, file_folder=file_folder)

app/config.py

@@ -59,6 +59,10 @@ class ConfigBcp:
     file: str = config["bcp"].get("file")
     pattern: str = config["bcp"].get("pattern")
     start_at: str = config["bcp"].get("start_at")
+    in_folder: bool = config["bcp"].getboolean("in_folder")
+    if in_folder:
+        folder_name_pattern: str = config["bcp"].get("folder_name_pattern")
+    min_string_count: int = config["bcp"].getint("min_string_count")
 
 
 @dataclass
@@ -81,6 +85,7 @@ class ConfigGit:
     protocol: str = config["git"].get("protocol")
     username: str = os.getenv("GIT_USERNAME")
     token: str = os.getenv("GIT_TOKEN")
+    remove_local: bool = config["git"].getboolean("remove_local")
 
 @dataclass
 class ConfigMail:

app/config/config-template.ini

@@ -27,6 +27,8 @@ protocol: https
 remote: gitlab.example.com.ru/secure/backup-network-device.git
 # ветка
 branch: main
+# Удаление локального репозитория после push
+remove_local: False
 
 [bcp]
 # Файл со списком устройств. Должен лежать в папке config
@@ -35,6 +37,12 @@ file: dev_list.txt
 pattern:(?P<name>\S+) (?P<ip>\d+\.\d+\.\d+\.\d+) (?P<vendor>\S+) (?P<model>\S+)
 # Во сколько будет делаться бэкап
 start_at: 18:59
+# True \ False - Создание папок внутри репозитория. Если False, то все файлы будут сохраняться в корень
+in_folder: True
+# Регулярное выражение по которому будет парситься имя папки из названия девайса
+folder_name_pattern: ^(.*?)-
+# Если количество строк в конфиге меньше указанного значения - ошибка, изменения не записываются.
+min_string_count: 50
 
 [net_dev]
 debug: False

app/main.py

@@ -8,7 +8,7 @@ from backup import (
     save_cisco_bcp,
 )
 from messages import send_mail
-from repo import check_bcp_repo, push_to_remote
+from repo import check_bcp_repo, push_to_remote, delete_all_files
 import schedule
 import time
 import os
@@ -48,6 +48,8 @@ def main():
                 save_cisco_bcp(host=current_device["ip"], name=current_device["name"])
     if cfg.git.push:
         push_to_remote()
+        if cfg.git.remove_local:
+            delete_all_files()
     if cfg.mail.send:
         if os.path.isfile(log_file):
             with open(log_file, 'r', encoding='utf-8') as file:

app/repo.py

@@ -1,8 +1,10 @@
-from git import Repo, InvalidGitRepositoryError, Git
+from git import Repo, InvalidGitRepositoryError, Git, rmtree
 from config import cfg
 import logging as log
 from datetime import datetime
-
+import os
+import shutil
+import stat
 
 def check_remote_repo(repo: Repo):
     log.info("check remote repo ")
@@ -52,9 +54,11 @@ def check_bcp_repo(path: str = cfg.bcp.dir) -> None:
         log.warning("Error: %r", e)
 
 
-def add_file_and_commit(file_name: str) -> None:
+def add_file_and_commit(file_name: str, file_folder: str) -> None:
     current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     repo = Repo(cfg.bcp.dir)
+    if file_folder is not None:
+        file_name = file_folder + "/" + file_name
     if file_name in repo.untracked_files:
         repo.git.add(file_name)
         repo.index.commit("add  %s %s" % (file_name, current_time))
@@ -68,8 +72,34 @@ def add_file_and_commit(file_name: str) -> None:
         log.info("no changed in file %r", file_name)
 
 
+
+
 def push_to_remote():
     repo = Repo(cfg.bcp.dir)
     remote = repo.remote(name=cfg.git.branch)
     remote.push(refspec=f"{cfg.git.branch}:{cfg.git.branch}")
     log.info("push to remote repo %r", cfg.git.branch)
+
+
+def delete_all_files(path: str = cfg.bcp.dir) -> None:
+
+    if os.path.exists(path) and os.path.isdir(path):
+        for filename in os.listdir(path):
+            file_path = os.path.join(path, filename)
+            if filename == ".git":
+                log.info('start remove read only for .git')
+                for root, dirs, files in os.walk(file_path):
+                    os.chmod(root, stat.S_IWRITE)
+                    for file in files:
+                        d_file_path = os.path.join(root, file)
+                        os.chmod(d_file_path, stat.S_IWRITE)
+                log.info('end remove read only for .git')
+            try:
+                if os.path.isfile(file_path) or os.path.islink(file_path):
+                    os.remove(file_path)
+                    log.info("Successfully deleting: %s", file_path)
+                elif os.path.isdir(file_path):
+                    shutil.rmtree(file_path)
+                    log.info("Successfully deleting: %s", file_path)
+            except Exception as e:
+                log.warning("Error deleting %s: %s", file_path, e)

docker/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: '3.3'
 services:
   net-backup:
-    image: git.sm8255082.ru/osnova/net-backup:1.1.6
+    image: git.sm8255082.ru/osnova/net-backup:1.2.0
     restart: "no"
     volumes:
       - ./logs:/app/logs