Merge pull request #13 from pmpbaptista/pb/cleanup

Update replace logic for copy and delete

README.md

@@ -4,6 +4,8 @@
 # folder-replicator
 Python utility for file backup and synchronization.
 
+[folder-replicator Documentation](https://pmpbaptista.github.io/folder-replicator/)
+
 ## Installation
 ```bash
 poetry install
@@ -14,11 +16,14 @@ poetry install
 poetry run folder-replicator --help
 ```
 
+## Docker Usage
+Edit the `docker-compose.yml` file to set the source and destination folders.
+```bash
+docker compose up build
+```
+
 ## Development
 ```bash
 poetry install
 poetry run tox
 ```
-
-## License
-[MIT](LICENSE)

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       dockerfile: Dockerfile
     image: folder-replicator:latest
     volumes:
-      - ./backup/source:/app/source # Edit source_path to the path of the source folder
-      - ./backup/dest:/app/destination # Edit destination_path to the path of the destination folder
+      - ./source_path:/app/source # Edit source_path to the path of the source folder
+      - ./destination_path:/app/destination # Edit destination_path to the path of the destination folder
     entrypoint: ["folder-replicator", "-s" ,"/app/source", "-d", "/app/destination", "-c", "* * * * *"] # Edit the cron schedule
     restart: unless-stopped

folder_replicator/Folder.py

@@ -72,7 +72,8 @@ def __calculate_folder_hash(self) -> str:
             str: the hash of the folder
         """
         hasher = hashlib.md5()
-        for _, hash in self.files.items():
+        for file, hash in self.files.items():
+            hasher.update(file.name.encode())
             hasher.update(hash.__str__().encode())
             hasher.update(hash.encode())
         return hasher.hexdigest()

folder_replicator/SyncStrategyLocal.py

@@ -48,6 +48,15 @@ def sync(self) -> None:
         for path, hash in source_files.items():
             if hash not in destination_files.values():
                 files_to_copy[path.resolve()] = hash
+                continue
+            for file, file_hash in destination_files.items():
+                if hash == file_hash:
+                    if path.name != file.name:
+                        print(path.name, file.name)
+                        files_to_copy[path.resolve()] = hash
+                        continue
+                    if path.stat().st_mtime > file.stat().st_mtime:
+                        files_to_copy[path.resolve()] = hash
 
         if logger.isEnabledFor(fr_logger.logging.DEBUG):
             logger.debug(f"Source files: {source_files}")
@@ -60,6 +69,14 @@ def sync(self) -> None:
             for path, hash in destination_files.items():
                 if hash not in source_files.values():
                     files_to_delete[path.resolve()] = hash
+                    continue
+                for file, file_hash in source_files.items():
+                    if hash == file_hash:
+                        if path.name != file.name:
+                            files_to_delete[path.resolve()] = hash
+                            continue
+                        if path.stat().st_mtime > file.stat().st_mtime:
+                            files_to_delete[path.resolve()] = hash
 
             if logger.isEnabledFor(fr_logger.logging.DEBUG):
                 logger.debug(f"Files to delete: {files_to_delete}")
@@ -92,10 +109,38 @@ def sync(self) -> None:
         if self.source.delete:
             # Clean up temp files
             self.__clean_bak_files(temp_files)
+            self.__remove_empty_folders(self.destination.path)
 
         self.source.refresh()
         self.destination.refresh()
         logger.info("Sync complete")
+
+    def __copy_file(self, src, dst, buffer_size=1024*1024):
+        """
+        Copy a file from source to destination using a buffer.
+
+        Args:
+            src: str - the source file path
+            dst: str - the destination file path
+            buffer_size: int - the buffer size to use
+
+        Returns:
+            None
+        """
+        copied_size = 0
+        logger = fr_logger.get_logger()
+
+        with open(src, 'rb') as file_src, open(dst, 'wb') as file_dst:
+            while True:
+                buf = file_src.read(buffer_size)
+                if not buf:
+                    break
+                file_dst.write(buf)
+                copied_size += len(buf)
+
+        shutil.copystat(src, dst)
+        if logger.isEnabledFor(fr_logger.logging.DEBUG):
+            logger.debug(f"Copied {copied_size} bytes from {src} to {dst}")
 
     def __copy(self, files_to_copy: dict) -> None:
         """
@@ -132,7 +177,8 @@ def __copy(self, files_to_copy: dict) -> None:
                 destination_path.parent.mkdir(parents=True, exist_ok=True)
 
                 # Copy the file
-                shutil.copy2(source_path, destination_path)
+                # shutil.copy2(source_path, destination_path)
+                self.__copy_file(source_path, destination_path)
             except FileNotFoundError:
                 logger.error(f"File {source_path} not found in source")
                 continue
@@ -177,3 +223,28 @@ def __clean_bak_files(self, temp_files: list) -> None:
                 continue
 
         logger.info("Temp file cleanup complete")
+
+    def __remove_empty_folders(self, path: Path) -> None:
+        """
+        Helper method to remove empty folders.
+
+        Args:
+            path: Path - the path to the folder
+
+        Returns:
+            None
+        """
+        logger = fr_logger.get_logger()
+        logger.info(f"Removing empty folders in {path}")
+
+        for folder in path.iterdir():
+            if folder.is_dir():
+                self.__remove_empty_folders(folder)
+                try:
+                    folder.rmdir()  # This will only remove the subdirectory if it is empty
+                    logger.info(f"Folder {folder} deleted")
+                except OSError as e:
+                    if e.errno == 39:  # Directory not empty
+                        logger.debug(f"Folder {folder} is not empty")
+                    else:
+                        logger.error(f"Error deleting folder {folder}: {e}")

pyproject.toml

@@ -90,6 +90,8 @@ exclude_also = [
     "def __init__",
     "except PermissionError",
     "except Exception",
+    "except FileNotFoundError",
+    "except OsError",
     ]
 
 [lint.per-file-ignores]

tests/test_SyncStrategyLocal.py

@@ -192,4 +192,30 @@ def test_sync_strategy_local_filenotfounderror(tmp_path):
     assert sync_strategy_local.source.hash != sync_strategy_local.destination.hash
     sync_strategy_local.source.path = tmp_path / "wrong"
     sync_strategy_local.sync()
-
+
+
+def test_sync_strategy_local_remove_empty_folders(tmp_path):
+    """
+    Test the SyncStrategyLocal class when removing empty folders.
+
+    Args:
+        tmp_path: Path - the temporary path to use for testing
+
+    Returns:
+        None
+    """
+    sync_strategy_local = setup_sync_strategy_local(tmp_path)
+    # Create a subfolder in the source folder
+    (sync_strategy_local.source.path / "test").mkdir()
+    # Create a file in the source folder
+    (sync_strategy_local.source.path / "test/file.txt").touch()
+    sync_strategy_local.source.refresh()
+    assert sync_strategy_local.source.hash != sync_strategy_local.destination.hash
+    sync_strategy_local.sync()
+    assert sync_strategy_local.source.hash == sync_strategy_local.destination.hash
+    # Remove the file in the source folder
+    (sync_strategy_local.source.path / "test/file.txt").unlink()
+    sync_strategy_local.sync()
+    sync_strategy_local.source.refresh()
+    assert not (sync_strategy_local.destination.path / "test").exists()
+    assert not (sync_strategy_local.destination.path / "test/file.txt").exists()