✨feature: 实现对备份文件的后台清理功能

Neeky · Jun 21, 2023 · 5253b53 · 5253b53
1 parent 53c1000
commit 5253b53
Show file tree

Hide file tree

Showing 6 changed files with 100 additions and 16 deletions.
diff --git a/dbma/bil/fs.py b/dbma/bil/fs.py
@@ -196,10 +196,10 @@ def truncate_or_delete_file(file_path: Path = None, chunk_size: int = 16 * 1024
     if file_size < chunk_size:
         os.remove(file_path)
         return 0
-    else:
-        chunk = file_size - chunk_size
-        os.truncate(file_path, chunk)
-        return chunk
+
+    chunk = file_size - chunk_size
+    os.truncate(file_path, chunk)
+    return chunk
 
 
 join = os.path.join

diff --git a/dbma/components/mysql/backends/clears.py b/dbma/components/mysql/backends/clears.py
@@ -14,9 +14,11 @@
     
 """
 
+import os
 import re
 import glob
 import time
+import shutil
 import logging
 from pathlib import Path
 from datetime import datetime
@@ -26,6 +28,8 @@
 from dbma.bil.fs import truncate_or_delete_file, get_file_size
 from dbma.core import messages
 from dbma.core.configs import dbm_agent_config
+from dbma.core.threads.backends import threads, keep_threads_running
+from dbma.bil.osuser import sudo
 
 
 clear_tasks = deque(maxlen=1024)
@@ -110,34 +114,98 @@ def clear_instance(task: ClearTask = None):
     """
     logging.info(messages.FUN_STARTS.format(fname()))
     logging.info(
-        "task.path = '{}'  is_expire = '{}' ".format(task.path, task.is_expired)
+        "task.path = '{}'  is_expire = '{}' ".format(task.path, task.is_expired())
     )
     files = []
     dirs = []
 
-    for item in glob.glob(task.path, recursive=True):
+    for item in glob.glob("{}/*".format(task.path)):
         path = Path(item)
         if path.is_dir():
-            dirs.append(dirs)
+            dirs.append(path)
         else:
             files.append(path)
+    logging.info("files = {}, dirs = {}".format(len(files), len(dirs)))
 
     # 先清理文件
     for path in files:
+        logging.info("deal-with file '{}' ".format(path))
         # 准备清理
         while True:
             # 如果文件比较大，那么就一直 truncate 到 0 为止
-            logging.info(
-                "prepare clear '{}' size = {}".format(path), get_file_size(path)
-            )
             chunck = truncate_or_delete_file(path, 16 * 1024 * 1024)
-            logging.info("done clear '{}' size = {}".format(path), get_file_size(path))
+            time.sleep(1)
             if chunck == 0:
+                # chunck == 0 说明文件已经执行 remove 清理掉了
+                logging.info("file '{}' removed ".format(path))
                 break
-        # 清理完成
-        time.sleep(1)
+            else:
+                logging.info("file '{}' truncated ".format(path))
 
-    # 清理目录
-    # TODO
+    # 清理子目录
+    for sub in dirs:
+        clear_instance(ClearTask(sub))
+
+    # 清理当前目录
+    dirs = []
+    for item in glob.glob("{}/*".format(task.path)):
+        path = Path(item)
+        if path.is_dir():
+            dirs.append(path)
+
+    if len(dirs) == 0:
+        logging.info(
+            "sub directorys not exists, rm current directory '{}' ".format(task.path)
+        )
+        shutil.rmtree(task.path)
 
     logging.info(messages.FUN_ENDS.format(fname()))
+
+
+def pub_clear_task_thread_fun():
+    """
+    生成后台清理任务的线程函数
+    """
+    global keep_threads_running
+    while keep_threads_running:
+        logging.info(messages.FUN_STARTS.format(fname()))
+        with sudo():
+            tasks = scan_data_dir_gen_task()
+            for task in tasks:
+                clear_tasks.append(task)
+
+        logging.info(messages.FUN_STARTS.format(fname()))
+
+        # 一小时扫一次目录，生成清理任务
+        time.sleep(3600)
+
+
+def sub_clear_task_thread_fun():
+    """
+    从队列里取出任务并执行清理
+    """
+    global keep_threads_running
+    while keep_threads_running:
+        try:
+            logging.info(messages.FUN_STARTS.format(fname()))
+            try:
+                task = clear_tasks.pop()
+            except IndexError as err:
+                logging.info("task deque is empty .")
+                # 对于队列中没有任务的情况下线程休息 30 分钟
+                time.sleep(1800)
+                continue
+
+            with sudo():
+                clear_instance(task)
+            # 清理完成一个实例之后要休息 1 分钟
+            time.sleep(60)
+            logging.info(messages.FUN_STARTS.format(fname()))
+        except Exception as err:
+            logging.error(err)
+
+
+def start_clear_tasks():
+    threads.submit(pub_clear_task_thread_fun)
+    time.sleep(5)
+    threads.submit(sub_clear_task_thread_fun)
diff --git a/dbma/core/httpserver.py b/dbma/core/httpserver.py
@@ -15,6 +15,7 @@
 from dbma.core.threads import backends
 from dbma.core.configs import DBMAgentConfig
 from dbma.core.views import dbmagentview as _
+from dbma.components.mysql.backends.clears import start_clear_tasks
 from dbma.components.mysql.views import defaultsview as _
 
 dbm_agent_config = DBMAgentConfig()
@@ -95,6 +96,7 @@ def start():
     # 启动后台线程
     logging.info("start backends threads .")
     backends.start_cycle_tasks()
+    start_clear_tasks()
 
     # 启动 http 服务
     logging.info("going to start dbm-agent http-server bind on 0.0.0.0:8086 .")

diff --git a/dbma/version.py b/dbma/version.py
@@ -7,7 +7,7 @@
 
 DBM_AGENT_MAJOR_VERSION = 8
 DBM_AGENT_MINOR_VERSION = 33
-DBM_AGENT_PATCH_VERSION = 11
+DBM_AGENT_PATCH_VERSION = 12
 
 
 # dbm-agent 版本号

diff --git a/docs/05-auto-clear-backup-files.md b/docs/05-auto-clear-backup-files.md
@@ -0,0 +1,13 @@
+```
+[2023-06-21 14:16:15,149 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 115 line]  ~  starts clear_instance .
+[2023-06-21 14:16:15,150 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 116 line]  ~  task.path = '/database/mysql/data/3308-backup-2023-05-29T20-23-32-472128/#innodb_redo'  is_expire = 'True' 
+[2023-06-21 14:16:15,150 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 128 line]  ~  files = 32, dirs = 0
+[2023-06-21 14:16:15,150 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 132 line]  ~  deal-with file '/database/mysql/data/3308-backup-2023-05-29T20-23-32-472128/#innodb_redo/#ib_redo22_tmp' 
+
+[2023-06-21 14:17:17,177 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 132 line]  ~  deal-with file '/database/mysql/data/3308-backup-2023-05-29T20-23-32-472128/#innodb_redo/#ib_redo3_tmp' 
+[2023-06-21 14:17:18,177 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 143 line]  ~  file '/database/mysql/data/3308-backup-2023-05-29T20-23-32-472128/#innodb_redo/#ib_redo3_tmp' truncated 
+[2023-06-21 14:17:19,177 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 140 line]  ~  file '/database/mysql/data/3308-backup-2023-05-29T20-23-32-472128/#innodb_redo/#ib_redo3_tmp' removed 
+[2023-06-21 14:17:19,178 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 157 line]  ~  sub directorys not exists, rm current directory '/database/mysql/data/3308-backup-2023-05-29T20-23-32-472128/#innodb_redo' 
+[2023-06-21 14:17:19,179 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 162 line]  ~  ends clear_instance .
+[2023-06-21 14:17:19,179 INFO] - [backends_2] - [/usr/local/python/lib/python3.11/site-packages/dbma/components/mysql/backends/clears.py 157 line]  ~  sub directorys not exists, rm current directory '/database/mysql/data/3308-backup-2023-05-29T20-23-32-472128'
+```
diff --git a/setup.py b/setup.py
@@ -46,6 +46,7 @@ def get_version():
         "dbma/components/mysql",
         "dbma/components/mysql/views",
         "dbma/components/mysql/backups",
+        "dbma/components/mysql/backends",
         "dbma/components/redis",
         "dbma/components/orchestrator",
     ],