From 6605ef79d54ff2f7413ec43c9fb9a0f5f9035b2f Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 11 Dec 2020 23:22:00 +0000 Subject: [PATCH 01/37] remove our own fsevents module --- src/maestral/fsevents/__init__.py | 97 ------------------------------ src/maestral/fsevents/fsevents.py | 99 ------------------------------- src/maestral/fsevents/polling.py | 91 ---------------------------- src/maestral/sync.py | 3 +- 4 files changed, 2 insertions(+), 288 deletions(-) delete mode 100644 src/maestral/fsevents/__init__.py delete mode 100644 src/maestral/fsevents/fsevents.py delete mode 100644 src/maestral/fsevents/polling.py diff --git a/src/maestral/fsevents/__init__.py b/src/maestral/fsevents/__init__.py deleted file mode 100644 index f93cceb01..000000000 --- a/src/maestral/fsevents/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module provides custom event emitters for the :obj:`watchdog` package that sort -file system events in an order which can be applied to reproduce the new state from the -old state. This is only required for event emitters which internally use -:class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff` to generate file system -events. This includes the macOS FSEvents emitter and the Polling emitter but not inotify -emitters. - -Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, -the event types are categorised as follows: - -* Created event: The inode is unique to the new snapshot. The path may be unique to the - new snapshot or exist in both. In the second case, there will be a preceding Deleted - event or a Moved event with the path as starting point (the old item was deleted or - moved away). - -* Deleted event: The inode is unique to the old snapshot. The path may be unique to the - old snapshot or exist in both. In the second case, there will be a subsequent Created - event or a Moved event with the path as end point (something else was created at or - moved to the location). - -* Moved event: The inode exists in both snapshots but with different paths. - -* Modified event: The inode exists in both snapshots and the mtime or file size are - different. DirectorySnapshotDiff will always use the inode’s path from the old - snapshot. - -From the above classification, there can be at most two created/deleted/moved events -that share the same path in one snapshot diff: - - * Deleted(path1) + Created(path1) - * Moved(path1, path2) + Created(path1) - * Deleted(path1) + Moved(path0, path1) - -Any Modified event will come before a Moved event or stand alone. Modified events will -never be combined by themselves with created or deleted events because they require the -inode to be present in both snapshots. - -From the above, we can achieve correct ordering for unique path by always adding Deleted -events to the queue first, Modified events second, Moved events third and Created events -last: - - Deleted -> Modified -> Moved -> Created - -The ordering won’t be correct between unrelated paths and between files and folder. The -first does not matter for syncing. We solve the second by assuming that when a directory -is deleted, so are its children. And before a child is created, its parent dircetory -must exist. - -MovedEvents which are not unique (their paths appear in other events) will be split -into Deleted and Created events by Maestral. -""" - -import os -from typing import Union - -from watchdog.utils import platform # type: ignore -from watchdog.utils import UnsupportedLibc - - -if platform.is_darwin(): - from .fsevents import OrderedFSEventsObserver as Observer -elif platform.is_linux(): - try: - from watchdog.observers.inotify import InotifyObserver as Observer # type: ignore - except UnsupportedLibc: - from .polling import OrderedPollingObserver as Observer -else: - from watchdog.observers import Observer # type: ignore - - -# patch encoding / decoding of paths in watchdog - - -def _patched_decode(path: Union[str, bytes]) -> str: - if isinstance(path, bytes): - return os.fsdecode(path) - return path - - -def _patched_encode(path: Union[str, bytes]) -> bytes: - if isinstance(path, str): - return os.fsencode(path) - return path - - -try: - from watchdog.utils import unicode_paths -except ImportError: - pass -else: - unicode_paths.decode = _patched_decode - unicode_paths.encode = _patched_encode - - -__all__ = ["Observer"] diff --git a/src/maestral/fsevents/fsevents.py b/src/maestral/fsevents/fsevents.py deleted file mode 100644 index f9d98924b..000000000 --- a/src/maestral/fsevents/fsevents.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2011 Yesudeep Mangalapilly -# Copyright 2012 Google, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from watchdog.observers.fsevents import ( # type: ignore - FSEventsEmitter, - FSEventsObserver, - FileDeletedEvent, - FileModifiedEvent, - FileMovedEvent, - FileCreatedEvent, - DirDeletedEvent, - DirModifiedEvent, - DirMovedEvent, - DirCreatedEvent, - DEFAULT_OBSERVER_TIMEOUT, - BaseObserver, -) -from watchdog.utils.dirsnapshot import DirectorySnapshot - - -class OrderedFSEventsEmitter(FSEventsEmitter): - """Ordered file system event emitter for macOS - - This subclasses FSEventsEmitter to guarantee an order of events which can be applied - to reproduce the new state from the old state. - """ - - def queue_events(self, timeout): - with self._lock: - if not self.watch.is_recursive and self.watch.path not in self.pathnames: - return - new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive) - diff = new_snapshot - self.snapshot - - # add metadata modified events which will be missed by regular diff - try: - ctime_files_modified = set() - - for path in self.snapshot.paths & new_snapshot.paths: - if not self.snapshot.isdir(path): - if self.snapshot.inode(path) == new_snapshot.inode(path): - if ( - self.snapshot.stat_info(path).st_ctime - != new_snapshot.stat_info(path).st_ctime - ): - ctime_files_modified.add(path) - - files_modified = set(ctime_files_modified) | set(diff.files_modified) - except Exception as exc: - print(exc) - - # replace cached snapshot - self.snapshot = new_snapshot - - # Files. - for src_path in diff.files_deleted: - self.queue_event(FileDeletedEvent(src_path)) - for src_path in files_modified: - self.queue_event(FileModifiedEvent(src_path)) - for src_path, dest_path in diff.files_moved: - self.queue_event(FileMovedEvent(src_path, dest_path)) - for src_path in diff.files_created: - self.queue_event(FileCreatedEvent(src_path)) - - # Directories. - for src_path in diff.dirs_deleted: - self.queue_event(DirDeletedEvent(src_path)) - for src_path in diff.dirs_modified: - self.queue_event(DirModifiedEvent(src_path)) - for src_path, dest_path in diff.dirs_moved: - self.queue_event(DirMovedEvent(src_path, dest_path)) - for src_path in diff.dirs_created: - self.queue_event(DirCreatedEvent(src_path)) - - # free some memory - del diff - del files_modified - - -class OrderedFSEventsObserver(FSEventsObserver): - def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT): - BaseObserver.__init__( - self, emitter_class=OrderedFSEventsEmitter, timeout=timeout - ) diff --git a/src/maestral/fsevents/polling.py b/src/maestral/fsevents/polling.py deleted file mode 100644 index 3577714e0..000000000 --- a/src/maestral/fsevents/polling.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2011 Yesudeep Mangalapilly -# Copyright 2012 Google, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from watchdog.observers.polling import ( # type: ignore - PollingEmitter, - PollingObserver, - FileDeletedEvent, - FileModifiedEvent, - FileMovedEvent, - FileCreatedEvent, - DirDeletedEvent, - DirModifiedEvent, - DirMovedEvent, - DirCreatedEvent, - DEFAULT_OBSERVER_TIMEOUT, - BaseObserver, -) -from watchdog.utils.dirsnapshot import DirectorySnapshotDiff - - -class OrderedPollingEmitter(PollingEmitter): - """Ordered polling file system event emitter - - Platform-independent emitter that polls a directory to detect file system changes. - Events are emitted in an order which can be used to produce the new file system - state from the old one. - """ - - def queue_events(self, timeout): - # We don't want to hit the disk continuously. - # timeout behaves like an interval for polling emitters. - if self.stopped_event.wait(timeout): - return - - with self._lock: - if not self.should_keep_running(): - return - - # Get event diff between fresh snapshot and previous snapshot. - # Update snapshot. - try: - new_snapshot = self._take_snapshot() - except OSError: - self.queue_event(DirDeletedEvent(self.watch.path)) - self.stop() - return - - events = DirectorySnapshotDiff(self._snapshot, new_snapshot) - self._snapshot = new_snapshot - - # Files. - for src_path in events.files_deleted: - self.queue_event(FileDeletedEvent(src_path)) - for src_path in events.files_modified: - self.queue_event(FileModifiedEvent(src_path)) - for src_path, dest_path in events.files_moved: - self.queue_event(FileMovedEvent(src_path, dest_path)) - for src_path in events.files_created: - self.queue_event(FileCreatedEvent(src_path)) - - # Directories. - for src_path in events.dirs_deleted: - self.queue_event(DirDeletedEvent(src_path)) - for src_path in events.dirs_modified: - self.queue_event(DirModifiedEvent(src_path)) - for src_path, dest_path in events.dirs_moved: - self.queue_event(DirMovedEvent(src_path, dest_path)) - for src_path in events.dirs_created: - self.queue_event(DirCreatedEvent(src_path)) - - -class OrderedPollingObserver(PollingObserver): - def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT): - BaseObserver.__init__( - self, emitter_class=OrderedPollingEmitter, timeout=timeout - ) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 01dec8a70..434a274ee 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -49,6 +49,7 @@ import pathspec # type: ignore import dropbox # type: ignore from dropbox.files import Metadata, DeletedMetadata, FileMetadata, FolderMetadata # type: ignore +from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler # type: ignore from watchdog.events import ( EVENT_TYPE_CREATED, @@ -72,7 +73,7 @@ # local imports from . import notify from .config import MaestralConfig, MaestralState -from .fsevents import Observer +from .notify import MaestralDesktopNotifier from .constants import ( IDLE, SYNCING, From 9079098bc72fa1f7d6c18030ffe3a5c79613cd02 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 11 Dec 2020 23:24:17 +0000 Subject: [PATCH 02/37] [sync] detect deleted dropbox folder through DirDeletedEvent --- src/maestral/sync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 434a274ee..4e88eacb1 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -2140,6 +2140,9 @@ def _on_local_moved(self, event: SyncEvent) -> Optional[Metadata]: :raises MaestralApiError: For any issues when syncing the item. """ + if event.local_path_from == self.dropbox_path: + self.ensure_dropbox_folder_present() + # fail fast on badly decoded paths validate_encoding(event.local_path) @@ -2396,6 +2399,9 @@ def _on_local_deleted(self, event: SyncEvent) -> Optional[Metadata]: :raises MaestralApiError: For any issues when syncing the item. """ + if event.local_path == self.dropbox_path: + self.ensure_dropbox_folder_present() + if self.is_excluded_by_user(event.dbx_path): logger.debug( 'Not deleting "%s": is excluded by selective sync', event.dbx_path From 670866eedd3623520eda5c90e0e64c1b642053d0 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 7 Jan 2021 22:06:54 +0000 Subject: [PATCH 03/37] bump watchdog to branch with macos fixes --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bfef98682..f7ec2c8cf 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog>=0.10.0,<=0.10.3;sys_platform=='darwin'", - "watchdog>=0.10.0;sys_platform=='linux'", + "watchdog@git+https://github.com/CCP-Aporia/watchdog.git@handle-coalesced-fsevents", ] gui_requires = [ From 58eafd6e7f2373983b7b46003d979966bfad66fb Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 8 Jan 2021 16:42:51 +0000 Subject: [PATCH 04/37] [sync] add nice repr to `_Ignore` --- src/maestral/sync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 4e88eacb1..a3487e05c 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -187,6 +187,12 @@ def __init__( self.ttl = ttl self.recursive = recursive + def __repr__(self) -> str: + return ( + f"<{self.__class__.__name__}(event={self.event}, " + f"recursive={self.recursive}, ttl={self.ttl})>" + ) + class FSEventHandler(FileSystemEventHandler): """A local file event handler From 3ee23501318af157ca5c25fe1ef2446657b8d430 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 8 Jan 2021 16:52:24 +0000 Subject: [PATCH 05/37] [sync] ignore up to two FileCreatedEvents on move on macOS --- src/maestral/sync.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index a3487e05c..271f71c0d 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -85,6 +85,7 @@ EXCLUDED_DIR_NAMES, MIGNORE_FILE, FILE_CACHE, + IS_MACOS, ) from .errors import ( SyncError, @@ -3181,10 +3182,12 @@ def _on_remote_file(self, event: SyncEvent) -> Optional[SyncEvent]: else: preserve_permissions = False - ignore_events = [ - FileMovedEvent(tmp_fname, local_path), - FileCreatedEvent(local_path), # sometimes emitted on macOS - ] + ignore_events = [FileMovedEvent(tmp_fname, local_path)] + + if IS_MACOS: + # up to two created events are sometimes emitted on macOS + ignore_events.append(FileCreatedEvent(local_path)) + ignore_events.append(FileCreatedEvent(local_path)) if preserve_permissions: # ignore FileModifiedEvent when changing permissions From f048dd265707a4b2efe7ef401e5674779377f374 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 8 Jan 2021 17:10:22 +0000 Subject: [PATCH 06/37] [tests] minor cleanup --- tests/linked/test_sync.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index 9ef7a19bd..b9fe33351 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -238,8 +238,9 @@ def test_rapid_remote_changes(m): mode=WriteMode.update(md.rev), ) + # reset file content with open(resources + "/file.txt", "w") as f: - f.write("content") # reset file content + f.write("content") wait_for_idle(m) @@ -672,14 +673,14 @@ def test_mignore(m): os.mkdir(m.test_folder_local + "/foo") wait_for_idle(m) - assert not (m.client.get_metadata("/sync_tests/foo")) + assert not m.client.get_metadata("/sync_tests/foo") # 3) test that renaming an item excludes it move(m.test_folder_local + "/bar", m.test_folder_local + "/build") wait_for_idle(m) - assert not (m.client.get_metadata("/sync_tests/build")) + assert not m.client.get_metadata("/sync_tests/build") # 4) test that renaming an item includes it From e3cc731a2bff60f2ac36e16bcb346b916c4c4a53 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Mon, 18 Jan 2021 16:42:32 +0000 Subject: [PATCH 07/37] [sync] whitelist which event types are handled --- src/maestral/sync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 271f71c0d..80bbffcba 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -239,6 +239,9 @@ def __init__( self.file_event_types = file_event_types self.dir_event_types = dir_event_types + self.file_event_types = file_event_types + self.dir_event_types = dir_event_types + self._ignored_events = [] self.ignore_timeout = 2.0 self.local_file_event_queue = Queue() From 600e12bd4afc27db50e7ce7291f34189662c4cdd Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Mon, 18 Jan 2021 19:26:18 +0000 Subject: [PATCH 08/37] [tests] increase time to wait for idle this accommodates waiting for delayed events --- tests/linked/conftest.py | 2 +- tests/linked/test_sync.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/linked/conftest.py b/tests/linked/conftest.py index 6cb3bf8d6..3cc348bb1 100644 --- a/tests/linked/conftest.py +++ b/tests/linked/conftest.py @@ -130,7 +130,7 @@ def proxy(m): # helper functions -def wait_for_idle(m: Maestral, minimum: int = 4): +def wait_for_idle(m: Maestral, minimum: int = 6): """Blocks until Maestral instance is idle for at least `minimum` sec.""" t0 = time.time() diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index b9fe33351..02516303b 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -296,7 +296,7 @@ def test_folder_tree_created_remote(m): # test deleting remote tree m.client.remove("/sync_tests/nested_folder") - wait_for_idle(m, 10) + wait_for_idle(m, 15) assert_synced(m) assert_child_count(m, "/sync_tests", 0) From 151bad7089f2d3e6bb62ff426fb55bc8327f7f3c Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Mon, 18 Jan 2021 19:26:33 +0000 Subject: [PATCH 09/37] [tests] prefer creating folder remotely --- tests/linked/test_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index 02516303b..cb7897be7 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -364,7 +364,7 @@ def test_remote_file_replaced_by_folder_and_unsynced_local_changes(m): def test_remote_folder_replaced_by_file(m): """Tests the download sync when a folder is replaced by a file.""" - os.mkdir(m.test_folder_local + "/folder") + m.client.make_dir("/sync_tests/folder") wait_for_idle(m) m.stop_sync() From 07e963f609388ee991172b1993c94f1fe07bbeae Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Tue, 19 Jan 2021 11:58:03 +0000 Subject: [PATCH 10/37] [tests] increase test timeouts --- tests/linked/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/linked/conftest.py b/tests/linked/conftest.py index 3cc348bb1..cfd26c4ec 100644 --- a/tests/linked/conftest.py +++ b/tests/linked/conftest.py @@ -130,7 +130,7 @@ def proxy(m): # helper functions -def wait_for_idle(m: Maestral, minimum: int = 6): +def wait_for_idle(m: Maestral, minimum: int = 10): """Blocks until Maestral instance is idle for at least `minimum` sec.""" t0 = time.time() From dad62086a79c14f25605ea56c571a30382e456f8 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Tue, 19 Jan 2021 12:46:32 +0000 Subject: [PATCH 11/37] install watchdog from master branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7ec2c8cf..abe4a0823 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog@git+https://github.com/CCP-Aporia/watchdog.git@handle-coalesced-fsevents", + "watchdog@git+https://github.com/CCP-Aporia/watchdog", ] gui_requires = [ From 84e5791ded28e87f3a37111b5cff9d3ebe189b37 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Wed, 20 Jan 2021 22:25:40 +0000 Subject: [PATCH 12/37] [sync] remove unused import --- src/maestral/sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 80bbffcba..229c8e921 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -73,7 +73,6 @@ # local imports from . import notify from .config import MaestralConfig, MaestralState -from .notify import MaestralDesktopNotifier from .constants import ( IDLE, SYNCING, From d43aa389baab0ef183fa59095e1218bbadf9066c Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 21 Jan 2021 18:53:21 +0000 Subject: [PATCH 13/37] switch to main watchdog repo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abe4a0823..1e5c13369 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog@git+https://github.com/CCP-Aporia/watchdog", + "watchdog@git+https://github.com/gorakhargosh/watchdog", ] gui_requires = [ From ed912ee6a551336b329c51a2022b0e9739eca9d7 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Wed, 3 Feb 2021 21:21:29 +0000 Subject: [PATCH 14/37] [fsevents] use our own polling observer again --- src/maestral/fsevents/__init__.py | 67 +++++++++++++++++++++++ src/maestral/fsevents/polling.py | 91 +++++++++++++++++++++++++++++++ src/maestral/sync.py | 2 +- 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 src/maestral/fsevents/__init__.py create mode 100644 src/maestral/fsevents/polling.py diff --git a/src/maestral/fsevents/__init__.py b/src/maestral/fsevents/__init__.py new file mode 100644 index 000000000..20c32173b --- /dev/null +++ b/src/maestral/fsevents/__init__.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +This module provides a custom polling file system event emitter for the +:obj:`watchdog` package that sorts file system events in an order which can be applied +to reproduce the new state from the old state. This is only required for the polling +emitter which uses period directory snapshots and compares them with a +:class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff` to generate file system +events. + +Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, +the event types are categorised as follows: + +* Created event: The inode is unique to the new snapshot. The path may be unique to the + new snapshot or exist in both. In the second case, there will be a preceding Deleted + event or a Moved event with the path as starting point (the old item was deleted or + moved away). + +* Deleted event: The inode is unique to the old snapshot. The path may be unique to the + old snapshot or exist in both. In the second case, there will be a subsequent Created + event or a Moved event with the path as end point (something else was created at or + moved to the location). + +* Moved event: The inode exists in both snapshots but with different paths. + +* Modified event: The inode exists in both snapshots and the mtime or file size are + different. DirectorySnapshotDiff will always use the inode’s path from the old + snapshot. + +From the above classification, there can be at most two created/deleted/moved events +that share the same path in one snapshot diff: + + * Deleted(path1) + Created(path1) + * Moved(path1, path2) + Created(path1) + * Deleted(path1) + Moved(path0, path1) + +Any Modified event will come before a Moved event or stand alone. Modified events will +never be combined by themselves with created or deleted events because they require the +inode to be present in both snapshots. + +From the above, we can achieve correct ordering for unique path by always adding Deleted +events to the queue first, Modified events second, Moved events third and Created events +last: + + Deleted -> Modified -> Moved -> Created + +The ordering won’t be correct between unrelated paths and between files and folder. The +first does not matter for syncing. We solve the second by assuming that when a directory +is deleted, so are its children. And before a child is created, its parent directory +must exist. + +MovedEvents which are not unique (their paths appear in other events) will be split +into Deleted and Created events by Maestral. +""" + +from watchdog.utils import platform # type: ignore +from watchdog.utils import UnsupportedLibc + + +if platform.is_linux(): + try: + from watchdog.observers.inotify import InotifyObserver as Observer # type: ignore + except UnsupportedLibc: + from .polling import OrderedPollingObserver as Observer +else: + from watchdog.observers import Observer # type: ignore + +__all__ = ["Observer"] diff --git a/src/maestral/fsevents/polling.py b/src/maestral/fsevents/polling.py new file mode 100644 index 000000000..3577714e0 --- /dev/null +++ b/src/maestral/fsevents/polling.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011 Yesudeep Mangalapilly +# Copyright 2012 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watchdog.observers.polling import ( # type: ignore + PollingEmitter, + PollingObserver, + FileDeletedEvent, + FileModifiedEvent, + FileMovedEvent, + FileCreatedEvent, + DirDeletedEvent, + DirModifiedEvent, + DirMovedEvent, + DirCreatedEvent, + DEFAULT_OBSERVER_TIMEOUT, + BaseObserver, +) +from watchdog.utils.dirsnapshot import DirectorySnapshotDiff + + +class OrderedPollingEmitter(PollingEmitter): + """Ordered polling file system event emitter + + Platform-independent emitter that polls a directory to detect file system changes. + Events are emitted in an order which can be used to produce the new file system + state from the old one. + """ + + def queue_events(self, timeout): + # We don't want to hit the disk continuously. + # timeout behaves like an interval for polling emitters. + if self.stopped_event.wait(timeout): + return + + with self._lock: + if not self.should_keep_running(): + return + + # Get event diff between fresh snapshot and previous snapshot. + # Update snapshot. + try: + new_snapshot = self._take_snapshot() + except OSError: + self.queue_event(DirDeletedEvent(self.watch.path)) + self.stop() + return + + events = DirectorySnapshotDiff(self._snapshot, new_snapshot) + self._snapshot = new_snapshot + + # Files. + for src_path in events.files_deleted: + self.queue_event(FileDeletedEvent(src_path)) + for src_path in events.files_modified: + self.queue_event(FileModifiedEvent(src_path)) + for src_path, dest_path in events.files_moved: + self.queue_event(FileMovedEvent(src_path, dest_path)) + for src_path in events.files_created: + self.queue_event(FileCreatedEvent(src_path)) + + # Directories. + for src_path in events.dirs_deleted: + self.queue_event(DirDeletedEvent(src_path)) + for src_path in events.dirs_modified: + self.queue_event(DirModifiedEvent(src_path)) + for src_path, dest_path in events.dirs_moved: + self.queue_event(DirMovedEvent(src_path, dest_path)) + for src_path in events.dirs_created: + self.queue_event(DirCreatedEvent(src_path)) + + +class OrderedPollingObserver(PollingObserver): + def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT): + BaseObserver.__init__( + self, emitter_class=OrderedPollingEmitter, timeout=timeout + ) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 229c8e921..5839bc4e3 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -49,7 +49,6 @@ import pathspec # type: ignore import dropbox # type: ignore from dropbox.files import Metadata, DeletedMetadata, FileMetadata, FolderMetadata # type: ignore -from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler # type: ignore from watchdog.events import ( EVENT_TYPE_CREATED, @@ -115,6 +114,7 @@ ItemType, ChangeType, ) +from .fsevents import Observer from .utils import removeprefix, sanitize_string from .utils.caches import LRUCache from .utils.path import ( From 475b7fb56ee8bb85ff4f5a22a2c25ebb1fb96df5 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Wed, 3 Feb 2021 22:45:32 +0000 Subject: [PATCH 15/37] [sync] remove special event ignores for macOS --- src/maestral/sync.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 5839bc4e3..89cfc8c76 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -83,7 +83,6 @@ EXCLUDED_DIR_NAMES, MIGNORE_FILE, FILE_CACHE, - IS_MACOS, ) from .errors import ( SyncError, @@ -3186,11 +3185,6 @@ def _on_remote_file(self, event: SyncEvent) -> Optional[SyncEvent]: ignore_events = [FileMovedEvent(tmp_fname, local_path)] - if IS_MACOS: - # up to two created events are sometimes emitted on macOS - ignore_events.append(FileCreatedEvent(local_path)) - ignore_events.append(FileCreatedEvent(local_path)) - if preserve_permissions: # ignore FileModifiedEvent when changing permissions ignore_events.append(FileModifiedEvent(local_path)) From 35cb31c3ae741adcca8175256b308ce1f4357cbe Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Wed, 3 Feb 2021 21:34:22 +0000 Subject: [PATCH 16/37] [sync] simplify notification for local events --- src/maestral/sync.py | 75 ++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 89cfc8c76..43f06f1d0 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -378,10 +378,42 @@ def on_any_event(self, event: FileSystemEvent) -> None: if self._is_ignored(event): return + self.queue_event(event) + + def queue_event(self, event: FileSystemEvent) -> None: + """ + Queues an individual file system event. Notifies / wakes up all threads that are + waiting with :meth:`wait_for_event`. + + :param event: File system event to queue. + """ with self.has_events: self.local_file_event_queue.put(event) self.has_events.notify_all() + def wait_for_event(self, timeout: float = 40) -> bool: + """ + Blocks until an event is available in the queue or a timeout occurs, whichever + comes first. You can use with method to wait for file system events in another + thread. + + .. note:: If there are multiple threads waiting for events, all of them will be + notified. If one of those threads starts getting events from + :attr:`local_file_event_queue`, other threads may find that queue empty. You + should therefore always be prepared to handle an empty queue, if if this + method returns ``True``. + + :param timeout: Maximum time to block in seconds. + :returns: ``True`` if an event is available, ``False`` if the call returns due + to a timeout. + """ + + with self.has_events: + if self.local_file_event_queue.qsize() > 0: + return True + self.has_events.wait(timeout) + return self.local_file_event_queue.qsize() > 0 + class PersistentStateMutableSet(abc.MutableSet): """Wraps a list in our state file as a MutableSet @@ -1613,13 +1645,7 @@ def wait_for_local_changes(self, timeout: float = 40) -> bool: logger.debug("Waiting for local changes since cursor: %s", self.local_cursor) - if self.fs_events.local_file_event_queue.qsize() > 0: - return True - - with self.fs_events.has_events: - self.fs_events.has_events.wait(timeout) - - return self.fs_events.local_file_event_queue.qsize() > 0 + return self.fs_events.wait_for_event(timeout) def upload_sync_cycle(self): """ @@ -3353,7 +3379,7 @@ def rescan(self, local_path: str) -> None: logger.debug('Rescanning "%s"', local_path) if osp.isfile(local_path): - self.fs_events.local_file_event_queue.put(FileModifiedEvent(local_path)) + self.fs_events.queue_event(FileModifiedEvent(local_path)) elif osp.isdir(local_path): # add created and deleted events of children as appropriate @@ -3364,9 +3390,9 @@ def rescan(self, local_path: str) -> None: for path in snapshot.paths: if snapshot.isdir(path): - self.fs_events.local_file_event_queue.put(DirCreatedEvent(path)) + self.fs_events.queue_event(DirCreatedEvent(path)) else: - self.fs_events.local_file_event_queue.put(FileModifiedEvent(path)) + self.fs_events.queue_event(FileModifiedEvent(path)) # add deleted events @@ -3377,22 +3403,16 @@ def rescan(self, local_path: str) -> None: .all() ) + dbx_root_lower = self.dropbox_path.lower() + for entry in entries: - child_path_uncased = ( - f"{self.dropbox_path}{entry.dbx_path_lower}".lower() - ) + child_path_uncased = f"{dbx_root_lower}{entry.dbx_path_lower}" if child_path_uncased not in lowercase_snapshot_paths: - local_child_path = self.to_local_path_from_cased( - entry.dbx_path_cased - ) + local_child = self.to_local_path_from_cased(entry.dbx_path_cased) if entry.is_directory: - self.fs_events.local_file_event_queue.put( - DirDeletedEvent(local_child_path) - ) + self.fs_events.queue_event(DirDeletedEvent(local_child)) else: - self.fs_events.local_file_event_queue.put( - FileDeletedEvent(local_child_path) - ) + self.fs_events.queue_event(FileDeletedEvent(local_child)) elif not osp.exists(local_path): dbx_path = self.to_dbx_path(local_path) @@ -3401,16 +3421,9 @@ def rescan(self, local_path: str) -> None: if local_entry: if local_entry.is_directory: - self.fs_events.local_file_event_queue.put( - DirDeletedEvent(local_path) - ) + self.fs_events.queue_event(DirDeletedEvent(local_path)) else: - self.fs_events.local_file_event_queue.put( - FileDeletedEvent(local_path) - ) - - with self.fs_events.has_events: - self.fs_events.has_events.notify_all() + self.fs_events.queue_event(FileDeletedEvent(local_path)) def _clean_history(self): """Commits new events and removes all events older than ``_keep_history`` from From 7eb78b8e0cbafdec560b76c545e4976a668957c0 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 4 Feb 2021 13:12:22 +0000 Subject: [PATCH 17/37] [sync] move FSEventHandler to SyncEngine --- src/maestral/main.py | 2 +- src/maestral/sync.py | 14 ++++++-------- tests/linked/test_sync.py | 2 +- tests/offline/conftest.py | 8 +++----- tests/offline/test_cleaning_events.py | 2 +- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/maestral/main.py b/src/maestral/main.py index a34acae56..8bde98e7d 100644 --- a/src/maestral/main.py +++ b/src/maestral/main.py @@ -1071,7 +1071,7 @@ def _remove_after_excluded(self, dbx_path: str) -> None: pass else: event_cls = DirDeletedEvent if osp.isdir(local_path) else FileDeletedEvent - with self.monitor.fs_event_handler.ignore(event_cls(local_path)): + with self.monitor.sync.fs_events.ignore(event_cls(local_path)): delete(local_path) def include_item(self, dbx_path: str) -> None: diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 43f06f1d0..90e00fd43 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -490,7 +490,6 @@ class SyncEngine: conflict resolution and updates to our index. :param client: Dropbox API client instance. - :param fs_events_handler: File system event handler to inform us of local events. """ sync_errors: Set[SyncError] @@ -500,11 +499,11 @@ class SyncEngine: _max_history = 1000 _num_threads = min(32, cpu_count * 3) - def __init__(self, client: DropboxClient, fs_events_handler: FSEventHandler): + def __init__(self, client: DropboxClient): self.client = client self.config_name = self.client.config_name - self.fs_events = fs_events_handler + self.fs_events = FSEventHandler() self.sync_lock = RLock() self._db_lock = RLock() @@ -3686,8 +3685,7 @@ def __init__(self, client: DropboxClient): self.added_item_queue = Queue() - self.fs_event_handler = FSEventHandler() - self.sync = SyncEngine(self.client, self.fs_event_handler) + self.sync = SyncEngine(self.client) self._startup_time = -1.0 @@ -3760,7 +3758,7 @@ def start(self) -> None: self.local_observer_thread = Observer(timeout=40) self.local_observer_thread.setName("maestral-fsobserver") self._watch = self.local_observer_thread.schedule( - self.fs_event_handler, self.sync.dropbox_path, recursive=True + self.sync.fs_events, self.sync.dropbox_path, recursive=True ) for i, emitter in enumerate(self.local_observer_thread.emitters): emitter.setName(f"maestral-fsemitter-{i}") @@ -3826,7 +3824,7 @@ def start(self) -> None: self.running.set() self.autostart.set() - self.fs_event_handler.enable() + self.sync.fs_events.enable() self.startup_thread.start() self.upload_thread.start() self.download_thread.start() @@ -3843,7 +3841,7 @@ def stop(self) -> None: logger.info("Shutting down threads...") - self.fs_event_handler.disable() + self.sync.fs_events.disable() self.running.clear() self.startup_completed.clear() self.autostart.clear() diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index cb7897be7..a21b5989b 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -191,7 +191,7 @@ def test_local_deletion_during_upload(m): # we mimic a deletion during upload by queueing a fake FileCreatedEvent fake_created_event = FileCreatedEvent(m.test_folder_local + "/file.txt") - m.monitor.fs_event_handler.local_file_event_queue.put(fake_created_event) + m.monitor.sync.fs_events.queue_event(fake_created_event) wait_for_idle(m) diff --git a/tests/offline/conftest.py b/tests/offline/conftest.py index a0c7059cb..dc1c8ca8e 100644 --- a/tests/offline/conftest.py +++ b/tests/offline/conftest.py @@ -7,7 +7,7 @@ import pytest from maestral.main import Maestral, logger -from maestral.sync import SyncEngine, Observer, FSEventHandler +from maestral.sync import SyncEngine, Observer from maestral.client import DropboxClient from maestral.config import list_configs, remove_configuration from maestral.daemon import stop_maestral_daemon_process, Stop @@ -32,10 +32,8 @@ def sync(): local_dir = osp.join(get_home_dir(), "dummy_dir") os.mkdir(local_dir) - fs_events_handler = FSEventHandler() - fs_events_handler.enable() - - sync = SyncEngine(DropboxClient("test-config"), fs_events_handler) + sync = SyncEngine(DropboxClient("test-config")) + sync.fs_events.enable() sync.dropbox_path = local_dir observer = Observer() diff --git a/tests/offline/test_cleaning_events.py b/tests/offline/test_cleaning_events.py index fe3ba97e0..5b608115a 100644 --- a/tests/offline/test_cleaning_events.py +++ b/tests/offline/test_cleaning_events.py @@ -20,7 +20,7 @@ @pytest.fixture def sync(): - sync = SyncEngine(DropboxClient("test-config"), None) + sync = SyncEngine(DropboxClient("test-config")) sync.dropbox_path = "/" yield sync From e005ee0c426a76045d50e4870162cfb93531a0f4 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 4 Feb 2021 19:22:08 +0000 Subject: [PATCH 18/37] [sync] added additional debug message --- src/maestral/sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 90e00fd43..1ad1d69d2 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -2667,6 +2667,7 @@ def list_remote_changes_iterator( # Pick up where we left off. This may be an interrupted indexing / # pagination through changes or a completely new set of changes. logger.info("Fetching remote changes...") + logger.debug("Fetching remote changes since cursor: %s", last_cursor) changes_iter = self.client.list_remote_changes_iterator(last_cursor) for changes in changes_iter: From 89691cd893392bfeb8707ad13d0a7d24468e63f5 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 4 Feb 2021 22:17:33 +0000 Subject: [PATCH 19/37] [sync] don't emit DirModifiedEvent on startup scan --- src/maestral/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 1ad1d69d2..876e2edd4 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -1600,8 +1600,8 @@ def _get_local_changes_while_inactive(self) -> Tuple[List[FileSystemEvent], floa elif is_modified: if snapshot.isdir(path) and index_entry.is_directory: # type: ignore - event = DirModifiedEvent(path) - changes.append(event) + # We don't emit `DirModifiedEvent`s. + pass elif not snapshot.isdir(path) and not index_entry.is_directory: # type: ignore event = FileModifiedEvent(path) changes.append(event) From 714656fde08a3d9337090b1849babf2139561434 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 4 Feb 2021 22:18:02 +0000 Subject: [PATCH 20/37] [database] show item type in SyncEvent repr --- src/maestral/database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/maestral/database.py b/src/maestral/database.py index cb51ae61f..b0218b6d3 100644 --- a/src/maestral/database.py +++ b/src/maestral/database.py @@ -280,7 +280,8 @@ def is_download(self) -> bool: def __repr__(self): return ( f"<{self.__class__.__name__}(direction={self.direction.name}, " - f"change_type={self.change_type.name}, dbx_path='{self.dbx_path}')>" + f"change_type={self.change_type.name}, item_type={self.item_type}, " + f"dbx_path='{self.dbx_path}')>" ) @classmethod From 3e1bfe21d60a583b41cd9bfc76a84fcc8415b34b Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 11 Dec 2020 23:22:00 +0000 Subject: [PATCH 21/37] remove our own fsevents module --- src/maestral/fsevents/__init__.py | 67 ----------------------- src/maestral/fsevents/polling.py | 91 ------------------------------- src/maestral/sync.py | 1 + 3 files changed, 1 insertion(+), 158 deletions(-) delete mode 100644 src/maestral/fsevents/__init__.py delete mode 100644 src/maestral/fsevents/polling.py diff --git a/src/maestral/fsevents/__init__.py b/src/maestral/fsevents/__init__.py deleted file mode 100644 index 20c32173b..000000000 --- a/src/maestral/fsevents/__init__.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -""" -This module provides a custom polling file system event emitter for the -:obj:`watchdog` package that sorts file system events in an order which can be applied -to reproduce the new state from the old state. This is only required for the polling -emitter which uses period directory snapshots and compares them with a -:class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff` to generate file system -events. - -Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, -the event types are categorised as follows: - -* Created event: The inode is unique to the new snapshot. The path may be unique to the - new snapshot or exist in both. In the second case, there will be a preceding Deleted - event or a Moved event with the path as starting point (the old item was deleted or - moved away). - -* Deleted event: The inode is unique to the old snapshot. The path may be unique to the - old snapshot or exist in both. In the second case, there will be a subsequent Created - event or a Moved event with the path as end point (something else was created at or - moved to the location). - -* Moved event: The inode exists in both snapshots but with different paths. - -* Modified event: The inode exists in both snapshots and the mtime or file size are - different. DirectorySnapshotDiff will always use the inode’s path from the old - snapshot. - -From the above classification, there can be at most two created/deleted/moved events -that share the same path in one snapshot diff: - - * Deleted(path1) + Created(path1) - * Moved(path1, path2) + Created(path1) - * Deleted(path1) + Moved(path0, path1) - -Any Modified event will come before a Moved event or stand alone. Modified events will -never be combined by themselves with created or deleted events because they require the -inode to be present in both snapshots. - -From the above, we can achieve correct ordering for unique path by always adding Deleted -events to the queue first, Modified events second, Moved events third and Created events -last: - - Deleted -> Modified -> Moved -> Created - -The ordering won’t be correct between unrelated paths and between files and folder. The -first does not matter for syncing. We solve the second by assuming that when a directory -is deleted, so are its children. And before a child is created, its parent directory -must exist. - -MovedEvents which are not unique (their paths appear in other events) will be split -into Deleted and Created events by Maestral. -""" - -from watchdog.utils import platform # type: ignore -from watchdog.utils import UnsupportedLibc - - -if platform.is_linux(): - try: - from watchdog.observers.inotify import InotifyObserver as Observer # type: ignore - except UnsupportedLibc: - from .polling import OrderedPollingObserver as Observer -else: - from watchdog.observers import Observer # type: ignore - -__all__ = ["Observer"] diff --git a/src/maestral/fsevents/polling.py b/src/maestral/fsevents/polling.py deleted file mode 100644 index 3577714e0..000000000 --- a/src/maestral/fsevents/polling.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2011 Yesudeep Mangalapilly -# Copyright 2012 Google, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from watchdog.observers.polling import ( # type: ignore - PollingEmitter, - PollingObserver, - FileDeletedEvent, - FileModifiedEvent, - FileMovedEvent, - FileCreatedEvent, - DirDeletedEvent, - DirModifiedEvent, - DirMovedEvent, - DirCreatedEvent, - DEFAULT_OBSERVER_TIMEOUT, - BaseObserver, -) -from watchdog.utils.dirsnapshot import DirectorySnapshotDiff - - -class OrderedPollingEmitter(PollingEmitter): - """Ordered polling file system event emitter - - Platform-independent emitter that polls a directory to detect file system changes. - Events are emitted in an order which can be used to produce the new file system - state from the old one. - """ - - def queue_events(self, timeout): - # We don't want to hit the disk continuously. - # timeout behaves like an interval for polling emitters. - if self.stopped_event.wait(timeout): - return - - with self._lock: - if not self.should_keep_running(): - return - - # Get event diff between fresh snapshot and previous snapshot. - # Update snapshot. - try: - new_snapshot = self._take_snapshot() - except OSError: - self.queue_event(DirDeletedEvent(self.watch.path)) - self.stop() - return - - events = DirectorySnapshotDiff(self._snapshot, new_snapshot) - self._snapshot = new_snapshot - - # Files. - for src_path in events.files_deleted: - self.queue_event(FileDeletedEvent(src_path)) - for src_path in events.files_modified: - self.queue_event(FileModifiedEvent(src_path)) - for src_path, dest_path in events.files_moved: - self.queue_event(FileMovedEvent(src_path, dest_path)) - for src_path in events.files_created: - self.queue_event(FileCreatedEvent(src_path)) - - # Directories. - for src_path in events.dirs_deleted: - self.queue_event(DirDeletedEvent(src_path)) - for src_path in events.dirs_modified: - self.queue_event(DirModifiedEvent(src_path)) - for src_path, dest_path in events.dirs_moved: - self.queue_event(DirMovedEvent(src_path, dest_path)) - for src_path in events.dirs_created: - self.queue_event(DirCreatedEvent(src_path)) - - -class OrderedPollingObserver(PollingObserver): - def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT): - BaseObserver.__init__( - self, emitter_class=OrderedPollingEmitter, timeout=timeout - ) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 876e2edd4..cdcde2b01 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -49,6 +49,7 @@ import pathspec # type: ignore import dropbox # type: ignore from dropbox.files import Metadata, DeletedMetadata, FileMetadata, FolderMetadata # type: ignore +from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler # type: ignore from watchdog.events import ( EVENT_TYPE_CREATED, From 98ab28621ba681cf9c7411c6f18724a07c9e775e Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 7 Jan 2021 22:06:54 +0000 Subject: [PATCH 22/37] bump watchdog to branch with macos fixes --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1e5c13369..f7ec2c8cf 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog@git+https://github.com/gorakhargosh/watchdog", + "watchdog@git+https://github.com/CCP-Aporia/watchdog.git@handle-coalesced-fsevents", ] gui_requires = [ From 4621fae106c326b36091358249a9e03cbc0d23a6 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 8 Jan 2021 16:52:24 +0000 Subject: [PATCH 23/37] [sync] ignore up to two FileCreatedEvents on move on macOS --- src/maestral/sync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index cdcde2b01..c23cfae4a 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -84,6 +84,7 @@ EXCLUDED_DIR_NAMES, MIGNORE_FILE, FILE_CACHE, + IS_MACOS, ) from .errors import ( SyncError, @@ -3212,6 +3213,11 @@ def _on_remote_file(self, event: SyncEvent) -> Optional[SyncEvent]: ignore_events = [FileMovedEvent(tmp_fname, local_path)] + if IS_MACOS: + # up to two created events are sometimes emitted on macOS + ignore_events.append(FileCreatedEvent(local_path)) + ignore_events.append(FileCreatedEvent(local_path)) + if preserve_permissions: # ignore FileModifiedEvent when changing permissions ignore_events.append(FileModifiedEvent(local_path)) From 63a4e58049f3d9efcbb401e585b0d28975e69a47 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Mon, 18 Jan 2021 19:26:18 +0000 Subject: [PATCH 24/37] [tests] increase time to wait for idle this accommodates waiting for delayed events --- tests/linked/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/linked/conftest.py b/tests/linked/conftest.py index cfd26c4ec..3cc348bb1 100644 --- a/tests/linked/conftest.py +++ b/tests/linked/conftest.py @@ -130,7 +130,7 @@ def proxy(m): # helper functions -def wait_for_idle(m: Maestral, minimum: int = 10): +def wait_for_idle(m: Maestral, minimum: int = 6): """Blocks until Maestral instance is idle for at least `minimum` sec.""" t0 = time.time() From 01e1a03dc7c92f5aa5ddb8195a9695c391e46555 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Tue, 19 Jan 2021 11:58:03 +0000 Subject: [PATCH 25/37] [tests] increase test timeouts --- tests/linked/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/linked/conftest.py b/tests/linked/conftest.py index 3cc348bb1..cfd26c4ec 100644 --- a/tests/linked/conftest.py +++ b/tests/linked/conftest.py @@ -130,7 +130,7 @@ def proxy(m): # helper functions -def wait_for_idle(m: Maestral, minimum: int = 6): +def wait_for_idle(m: Maestral, minimum: int = 10): """Blocks until Maestral instance is idle for at least `minimum` sec.""" t0 = time.time() From cd81374f9ff8cea9c3ec9f49f3357834bead36a9 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Tue, 19 Jan 2021 12:46:32 +0000 Subject: [PATCH 26/37] install watchdog from master branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7ec2c8cf..abe4a0823 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog@git+https://github.com/CCP-Aporia/watchdog.git@handle-coalesced-fsevents", + "watchdog@git+https://github.com/CCP-Aporia/watchdog", ] gui_requires = [ From d03e7bd17156fc6d651feef8ac12f987f7a11913 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 21 Jan 2021 18:53:21 +0000 Subject: [PATCH 27/37] switch to main watchdog repo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abe4a0823..1e5c13369 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog@git+https://github.com/CCP-Aporia/watchdog", + "watchdog@git+https://github.com/gorakhargosh/watchdog", ] gui_requires = [ From f49d1ac4b0b1fb6d02a9d255fac81d2b0756573c Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Wed, 3 Feb 2021 21:21:29 +0000 Subject: [PATCH 28/37] [fsevents] use our own polling observer again --- src/maestral/fsevents/__init__.py | 67 +++++++++++++++++++++++ src/maestral/fsevents/polling.py | 91 +++++++++++++++++++++++++++++++ src/maestral/sync.py | 1 - 3 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 src/maestral/fsevents/__init__.py create mode 100644 src/maestral/fsevents/polling.py diff --git a/src/maestral/fsevents/__init__.py b/src/maestral/fsevents/__init__.py new file mode 100644 index 000000000..20c32173b --- /dev/null +++ b/src/maestral/fsevents/__init__.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +This module provides a custom polling file system event emitter for the +:obj:`watchdog` package that sorts file system events in an order which can be applied +to reproduce the new state from the old state. This is only required for the polling +emitter which uses period directory snapshots and compares them with a +:class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff` to generate file system +events. + +Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, +the event types are categorised as follows: + +* Created event: The inode is unique to the new snapshot. The path may be unique to the + new snapshot or exist in both. In the second case, there will be a preceding Deleted + event or a Moved event with the path as starting point (the old item was deleted or + moved away). + +* Deleted event: The inode is unique to the old snapshot. The path may be unique to the + old snapshot or exist in both. In the second case, there will be a subsequent Created + event or a Moved event with the path as end point (something else was created at or + moved to the location). + +* Moved event: The inode exists in both snapshots but with different paths. + +* Modified event: The inode exists in both snapshots and the mtime or file size are + different. DirectorySnapshotDiff will always use the inode’s path from the old + snapshot. + +From the above classification, there can be at most two created/deleted/moved events +that share the same path in one snapshot diff: + + * Deleted(path1) + Created(path1) + * Moved(path1, path2) + Created(path1) + * Deleted(path1) + Moved(path0, path1) + +Any Modified event will come before a Moved event or stand alone. Modified events will +never be combined by themselves with created or deleted events because they require the +inode to be present in both snapshots. + +From the above, we can achieve correct ordering for unique path by always adding Deleted +events to the queue first, Modified events second, Moved events third and Created events +last: + + Deleted -> Modified -> Moved -> Created + +The ordering won’t be correct between unrelated paths and between files and folder. The +first does not matter for syncing. We solve the second by assuming that when a directory +is deleted, so are its children. And before a child is created, its parent directory +must exist. + +MovedEvents which are not unique (their paths appear in other events) will be split +into Deleted and Created events by Maestral. +""" + +from watchdog.utils import platform # type: ignore +from watchdog.utils import UnsupportedLibc + + +if platform.is_linux(): + try: + from watchdog.observers.inotify import InotifyObserver as Observer # type: ignore + except UnsupportedLibc: + from .polling import OrderedPollingObserver as Observer +else: + from watchdog.observers import Observer # type: ignore + +__all__ = ["Observer"] diff --git a/src/maestral/fsevents/polling.py b/src/maestral/fsevents/polling.py new file mode 100644 index 000000000..3577714e0 --- /dev/null +++ b/src/maestral/fsevents/polling.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2011 Yesudeep Mangalapilly +# Copyright 2012 Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from watchdog.observers.polling import ( # type: ignore + PollingEmitter, + PollingObserver, + FileDeletedEvent, + FileModifiedEvent, + FileMovedEvent, + FileCreatedEvent, + DirDeletedEvent, + DirModifiedEvent, + DirMovedEvent, + DirCreatedEvent, + DEFAULT_OBSERVER_TIMEOUT, + BaseObserver, +) +from watchdog.utils.dirsnapshot import DirectorySnapshotDiff + + +class OrderedPollingEmitter(PollingEmitter): + """Ordered polling file system event emitter + + Platform-independent emitter that polls a directory to detect file system changes. + Events are emitted in an order which can be used to produce the new file system + state from the old one. + """ + + def queue_events(self, timeout): + # We don't want to hit the disk continuously. + # timeout behaves like an interval for polling emitters. + if self.stopped_event.wait(timeout): + return + + with self._lock: + if not self.should_keep_running(): + return + + # Get event diff between fresh snapshot and previous snapshot. + # Update snapshot. + try: + new_snapshot = self._take_snapshot() + except OSError: + self.queue_event(DirDeletedEvent(self.watch.path)) + self.stop() + return + + events = DirectorySnapshotDiff(self._snapshot, new_snapshot) + self._snapshot = new_snapshot + + # Files. + for src_path in events.files_deleted: + self.queue_event(FileDeletedEvent(src_path)) + for src_path in events.files_modified: + self.queue_event(FileModifiedEvent(src_path)) + for src_path, dest_path in events.files_moved: + self.queue_event(FileMovedEvent(src_path, dest_path)) + for src_path in events.files_created: + self.queue_event(FileCreatedEvent(src_path)) + + # Directories. + for src_path in events.dirs_deleted: + self.queue_event(DirDeletedEvent(src_path)) + for src_path in events.dirs_modified: + self.queue_event(DirModifiedEvent(src_path)) + for src_path, dest_path in events.dirs_moved: + self.queue_event(DirMovedEvent(src_path, dest_path)) + for src_path in events.dirs_created: + self.queue_event(DirCreatedEvent(src_path)) + + +class OrderedPollingObserver(PollingObserver): + def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT): + BaseObserver.__init__( + self, emitter_class=OrderedPollingEmitter, timeout=timeout + ) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index c23cfae4a..9f135d10e 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -49,7 +49,6 @@ import pathspec # type: ignore import dropbox # type: ignore from dropbox.files import Metadata, DeletedMetadata, FileMetadata, FolderMetadata # type: ignore -from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler # type: ignore from watchdog.events import ( EVENT_TYPE_CREATED, From be39d621186d866bee286fc2ee455de801d0bca5 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Wed, 3 Feb 2021 22:45:32 +0000 Subject: [PATCH 29/37] [sync] remove special event ignores for macOS --- src/maestral/sync.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 9f135d10e..876e2edd4 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -83,7 +83,6 @@ EXCLUDED_DIR_NAMES, MIGNORE_FILE, FILE_CACHE, - IS_MACOS, ) from .errors import ( SyncError, @@ -3212,11 +3211,6 @@ def _on_remote_file(self, event: SyncEvent) -> Optional[SyncEvent]: ignore_events = [FileMovedEvent(tmp_fname, local_path)] - if IS_MACOS: - # up to two created events are sometimes emitted on macOS - ignore_events.append(FileCreatedEvent(local_path)) - ignore_events.append(FileCreatedEvent(local_path)) - if preserve_permissions: # ignore FileModifiedEvent when changing permissions ignore_events.append(FileModifiedEvent(local_path)) From 690d2a8d32cce7fa7cc5d71e6a7daa98862889f9 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 5 Feb 2021 01:14:48 +0000 Subject: [PATCH 30/37] [tests] reduce wait_for_idle time --- tests/linked/conftest.py | 2 +- tests/linked/test_sync.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/linked/conftest.py b/tests/linked/conftest.py index cfd26c4ec..d90922313 100644 --- a/tests/linked/conftest.py +++ b/tests/linked/conftest.py @@ -130,7 +130,7 @@ def proxy(m): # helper functions -def wait_for_idle(m: Maestral, minimum: int = 10): +def wait_for_idle(m: Maestral, minimum: int = 5): """Blocks until Maestral instance is idle for at least `minimum` sec.""" t0 = time.time() diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index a21b5989b..ea68a8180 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -521,6 +521,8 @@ def test_local_file_replaced_by_folder_and_unsynced_remote_changes(m): mode=WriteMode.overwrite, ) + wait_for_idle(m) + m.start_sync() wait_for_idle(m) From 0218f2662ef50abe604d783bbe8a3c233f4ef429 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 11 Feb 2021 13:49:59 +0000 Subject: [PATCH 31/37] bump to watchdog 2.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1e5c13369..617b46efc 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog@git+https://github.com/gorakhargosh/watchdog", + "watchdog>=2.0", ] gui_requires = [ From 78b262244993a10c9bedb7f7abd9d9be6ac3b514 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 11 Feb 2021 13:52:26 +0000 Subject: [PATCH 32/37] [fsevents] moved doc strings for polling emitter --- src/maestral/fsevents/__init__.py | 44 ---------------------------- src/maestral/fsevents/polling.py | 48 +++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/maestral/fsevents/__init__.py b/src/maestral/fsevents/__init__.py index 20c32173b..24102c880 100644 --- a/src/maestral/fsevents/__init__.py +++ b/src/maestral/fsevents/__init__.py @@ -6,50 +6,6 @@ emitter which uses period directory snapshots and compares them with a :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff` to generate file system events. - -Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, -the event types are categorised as follows: - -* Created event: The inode is unique to the new snapshot. The path may be unique to the - new snapshot or exist in both. In the second case, there will be a preceding Deleted - event or a Moved event with the path as starting point (the old item was deleted or - moved away). - -* Deleted event: The inode is unique to the old snapshot. The path may be unique to the - old snapshot or exist in both. In the second case, there will be a subsequent Created - event or a Moved event with the path as end point (something else was created at or - moved to the location). - -* Moved event: The inode exists in both snapshots but with different paths. - -* Modified event: The inode exists in both snapshots and the mtime or file size are - different. DirectorySnapshotDiff will always use the inode’s path from the old - snapshot. - -From the above classification, there can be at most two created/deleted/moved events -that share the same path in one snapshot diff: - - * Deleted(path1) + Created(path1) - * Moved(path1, path2) + Created(path1) - * Deleted(path1) + Moved(path0, path1) - -Any Modified event will come before a Moved event or stand alone. Modified events will -never be combined by themselves with created or deleted events because they require the -inode to be present in both snapshots. - -From the above, we can achieve correct ordering for unique path by always adding Deleted -events to the queue first, Modified events second, Moved events third and Created events -last: - - Deleted -> Modified -> Moved -> Created - -The ordering won’t be correct between unrelated paths and between files and folder. The -first does not matter for syncing. We solve the second by assuming that when a directory -is deleted, so are its children. And before a child is created, its parent directory -must exist. - -MovedEvents which are not unique (their paths appear in other events) will be split -into Deleted and Created events by Maestral. """ from watchdog.utils import platform # type: ignore diff --git a/src/maestral/fsevents/polling.py b/src/maestral/fsevents/polling.py index 3577714e0..3768674bf 100644 --- a/src/maestral/fsevents/polling.py +++ b/src/maestral/fsevents/polling.py @@ -1,6 +1,50 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -# +""" +Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, +the event types are categorised as follows: + +* Created event: The inode is unique to the new snapshot. The path may be unique to the + new snapshot or exist in both. In the second case, there will be a preceding Deleted + event or a Moved event with the path as starting point (the old item was deleted or + moved away). + +* Deleted event: The inode is unique to the old snapshot. The path may be unique to the + old snapshot or exist in both. In the second case, there will be a subsequent Created + event or a Moved event with the path as end point (something else was created at or + moved to the location). + +* Moved event: The inode exists in both snapshots but with different paths. + +* Modified event: The inode exists in both snapshots and the mtime or file size are + different. DirectorySnapshotDiff will always use the inode’s path from the old + snapshot. + +From the above classification, there can be at most two created/deleted/moved events +that share the same path in one snapshot diff: + + * Deleted(path1) + Created(path1) + * Moved(path1, path2) + Created(path1) + * Deleted(path1) + Moved(path0, path1) + +Any Modified event will come before a Moved event or stand alone. Modified events will +never be combined by themselves with created or deleted events because they require the +inode to be present in both snapshots. + +From the above, we can achieve correct ordering for unique path by always adding Deleted +events to the queue first, Modified events second, Moved events third and Created events +last: + + Deleted -> Modified -> Moved -> Created + +The ordering won’t be correct between unrelated paths and between files and folder. The +first does not matter for syncing. We solve the second by assuming that when a directory +is deleted, so are its children. And before a child is created, its parent directory +must exist. + +MovedEvents which are not unique (their paths appear in other events) will be split +into Deleted and Created events by Maestral. +""" + # Copyright 2011 Yesudeep Mangalapilly # Copyright 2012 Google, Inc. # From 4cbcacb9f184b949d98eb5dbf43434d8f65a2c7e Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 11 Feb 2021 14:43:58 +0000 Subject: [PATCH 33/37] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19b024918..6e15916a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ * Bumped survey to version >=3.2.2,<4.0. * Bumped keyring to version >=22. +* Bumped watchdog to version >= 2.0. * Added desktop-notifier dependency. This is spin-off from Maestral, built on the code from the `notify` module. * Removed bugsnag dependency. From a0d231ff473d0f565404abe82db6a4b1a43bb989 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Thu, 11 Feb 2021 22:50:00 +0000 Subject: [PATCH 34/37] bump desktop-notifier to 3.1.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 617b46efc..e6ff2bbdd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ install_requires = [ "alembic>=1.3", "click>=7.1.1", - "desktop-notifier>=3.1", + "desktop-notifier>=3.1.2", "dropbox>=10.9.0,<12.0", "fasteners>=0.15", "importlib_metadata;python_version<'3.8'", From 4417257e773310482ae04a6708ae3579f7b0bd9e Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 12 Feb 2021 01:11:51 +0000 Subject: [PATCH 35/37] [tests] record fsevents log --- tests/linked/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/linked/conftest.py b/tests/linked/conftest.py index d90922313..281621828 100644 --- a/tests/linked/conftest.py +++ b/tests/linked/conftest.py @@ -27,6 +27,8 @@ resources = os.path.dirname(__file__) + "/resources" +fsevents_logger = logging.getLogger("fsevents") +fsevents_logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG) From 4c81529d073da88e3bc7531ab946e56b3d35d4fb Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 12 Feb 2021 01:12:03 +0000 Subject: [PATCH 36/37] [tests] use sync_lock instead of start / stop --- tests/linked/test_sync.py | 107 ++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 62 deletions(-) diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index ea68a8180..1206e9d20 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -311,14 +311,12 @@ def test_remote_file_replaced_by_folder(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace remote file with folder - m.client.remove("/sync_tests/file.txt") - m.client.make_dir("/sync_tests/file.txt") + # replace remote file with folder + m.client.remove("/sync_tests/file.txt") + m.client.make_dir("/sync_tests/file.txt") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -338,18 +336,16 @@ def test_remote_file_replaced_by_folder_and_unsynced_local_changes(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace remote file with folder - m.client.remove("/sync_tests/file.txt") - m.client.make_dir("/sync_tests/file.txt") + # replace remote file with folder + m.client.remove("/sync_tests/file.txt") + m.client.make_dir("/sync_tests/file.txt") - # create local changes - with open(m.test_folder_local + "/file.txt", "a") as f: - f.write(" modified") + # create local changes + with open(m.test_folder_local + "/file.txt", "a") as f: + f.write(" modified") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -367,14 +363,12 @@ def test_remote_folder_replaced_by_file(m): m.client.make_dir("/sync_tests/folder") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) - # replace remote folder with file - m.client.remove("/sync_tests/folder") - m.client.upload(resources + "/file.txt", "/sync_tests/folder") - m.start_sync() + with m.sync.sync_lock: + m.client.remove("/sync_tests/folder") + m.client.upload(resources + "/file.txt", "/sync_tests/folder") + wait_for_idle(m) assert_synced(m) @@ -394,17 +388,15 @@ def test_remote_folder_replaced_by_file_and_unsynced_local_changes(m): os.mkdir(m.test_folder_local + "/folder") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace remote folder with file - m.client.remove("/sync_tests/folder") - m.client.upload(resources + "/file.txt", "/sync_tests/folder") + # replace remote folder with file + m.client.remove("/sync_tests/folder") + m.client.upload(resources + "/file.txt", "/sync_tests/folder") - # create local changes - os.mkdir(m.test_folder_local + "/folder/subfolder") + # create local changes + os.mkdir(m.test_folder_local + "/folder/subfolder") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -422,13 +414,12 @@ def test_local_folder_replaced_by_file(m): os.mkdir(m.test_folder_local + "/folder") wait_for_idle(m) - m.stop_sync() + with m.sync.sync_lock: - # replace local folder with file - delete(m.test_folder_local + "/folder") - shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") + # replace local folder with file + delete(m.test_folder_local + "/folder") + shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -450,17 +441,15 @@ def test_local_folder_replaced_by_file_and_unsynced_remote_changes(m): os.mkdir(m.test_folder_local + "/folder") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace local folder with file - delete(m.test_folder_local + "/folder") - shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") + # replace local folder with file + delete(m.test_folder_local + "/folder") + shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") - # create remote changes - m.client.upload(resources + "/file1.txt", "/sync_tests/folder/file.txt") + # create remote changes + m.client.upload(resources + "/file1.txt", "/sync_tests/folder/file.txt") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -477,14 +466,12 @@ def test_local_file_replaced_by_folder(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace local file with folder - os.unlink(m.test_folder_local + "/file.txt") - os.mkdir(m.test_folder_local + "/file.txt") + # replace local file with folder + os.unlink(m.test_folder_local + "/file.txt") + os.mkdir(m.test_folder_local + "/file.txt") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -507,23 +494,19 @@ def test_local_file_replaced_by_folder_and_unsynced_remote_changes(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace local file with folder - os.unlink(m.test_folder_local + "/file.txt") - os.mkdir(m.test_folder_local + "/file.txt") + # replace local file with folder + os.unlink(m.test_folder_local + "/file.txt") + os.mkdir(m.test_folder_local + "/file.txt") - # create remote changes - m.client.upload( - resources + "/file1.txt", - "/sync_tests/file.txt", - mode=WriteMode.overwrite, - ) - - wait_for_idle(m) + # create remote changes + m.client.upload( + resources + "/file1.txt", + "/sync_tests/file.txt", + mode=WriteMode.overwrite, + ) - m.start_sync() wait_for_idle(m) assert_synced(m) From 1640cee80fc97eda3a222a54cc6cb9c1f3de51e1 Mon Sep 17 00:00:00 2001 From: Sam Schott Date: Fri, 12 Feb 2021 11:48:34 +0000 Subject: [PATCH 37/37] slightly relax performance requirement GH runners on macOS are sometimes a bit slow... --- tests/linked/test_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index 1206e9d20..eaa013972 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -916,7 +916,7 @@ def generate_sync_events(): duration = timeit.timeit(stmt=generate_sync_events, setup=setup, number=n_loops) - assert duration < 3 # expected ~ 1.8 sec + assert duration < 4 # expected ~ 1.8 sec def test_invalid_pending_download(m):