diff --git a/CHANGELOG.md b/CHANGELOG.md index 19b024918..6e15916a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ * Bumped survey to version >=3.2.2,<4.0. * Bumped keyring to version >=22. +* Bumped watchdog to version >= 2.0. * Added desktop-notifier dependency. This is spin-off from Maestral, built on the code from the `notify` module. * Removed bugsnag dependency. diff --git a/setup.py b/setup.py index bfef98682..e6ff2bbdd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ install_requires = [ "alembic>=1.3", "click>=7.1.1", - "desktop-notifier>=3.1", + "desktop-notifier>=3.1.2", "dropbox>=10.9.0,<12.0", "fasteners>=0.15", "importlib_metadata;python_version<'3.8'", @@ -24,8 +24,7 @@ "setuptools", "sqlalchemy>=1.3", "survey>=3.2.2,<4.0", - "watchdog>=0.10.0,<=0.10.3;sys_platform=='darwin'", - "watchdog>=0.10.0;sys_platform=='linux'", + "watchdog>=2.0", ] gui_requires = [ diff --git a/src/maestral/database.py b/src/maestral/database.py index cb51ae61f..b0218b6d3 100644 --- a/src/maestral/database.py +++ b/src/maestral/database.py @@ -280,7 +280,8 @@ def is_download(self) -> bool: def __repr__(self): return ( f"<{self.__class__.__name__}(direction={self.direction.name}, " - f"change_type={self.change_type.name}, dbx_path='{self.dbx_path}')>" + f"change_type={self.change_type.name}, item_type={self.item_type}, " + f"dbx_path='{self.dbx_path}')>" ) @classmethod diff --git a/src/maestral/fsevents/__init__.py b/src/maestral/fsevents/__init__.py index f93cceb01..24102c880 100644 --- a/src/maestral/fsevents/__init__.py +++ b/src/maestral/fsevents/__init__.py @@ -1,67 +1,18 @@ # -*- coding: utf-8 -*- """ -This module provides custom event emitters for the :obj:`watchdog` package that sort -file system events in an order which can be applied to reproduce the new state from the -old state. This is only required for event emitters which internally use +This module provides a custom polling file system event emitter for the +:obj:`watchdog` package that sorts file system events in an order which can be applied +to reproduce the new state from the old state. This is only required for the polling +emitter which uses period directory snapshots and compares them with a :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff` to generate file system -events. This includes the macOS FSEvents emitter and the Polling emitter but not inotify -emitters. - -Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, -the event types are categorised as follows: - -* Created event: The inode is unique to the new snapshot. The path may be unique to the - new snapshot or exist in both. In the second case, there will be a preceding Deleted - event or a Moved event with the path as starting point (the old item was deleted or - moved away). - -* Deleted event: The inode is unique to the old snapshot. The path may be unique to the - old snapshot or exist in both. In the second case, there will be a subsequent Created - event or a Moved event with the path as end point (something else was created at or - moved to the location). - -* Moved event: The inode exists in both snapshots but with different paths. - -* Modified event: The inode exists in both snapshots and the mtime or file size are - different. DirectorySnapshotDiff will always use the inode’s path from the old - snapshot. - -From the above classification, there can be at most two created/deleted/moved events -that share the same path in one snapshot diff: - - * Deleted(path1) + Created(path1) - * Moved(path1, path2) + Created(path1) - * Deleted(path1) + Moved(path0, path1) - -Any Modified event will come before a Moved event or stand alone. Modified events will -never be combined by themselves with created or deleted events because they require the -inode to be present in both snapshots. - -From the above, we can achieve correct ordering for unique path by always adding Deleted -events to the queue first, Modified events second, Moved events third and Created events -last: - - Deleted -> Modified -> Moved -> Created - -The ordering won’t be correct between unrelated paths and between files and folder. The -first does not matter for syncing. We solve the second by assuming that when a directory -is deleted, so are its children. And before a child is created, its parent dircetory -must exist. - -MovedEvents which are not unique (their paths appear in other events) will be split -into Deleted and Created events by Maestral. +events. """ -import os -from typing import Union - from watchdog.utils import platform # type: ignore from watchdog.utils import UnsupportedLibc -if platform.is_darwin(): - from .fsevents import OrderedFSEventsObserver as Observer -elif platform.is_linux(): +if platform.is_linux(): try: from watchdog.observers.inotify import InotifyObserver as Observer # type: ignore except UnsupportedLibc: @@ -69,29 +20,4 @@ else: from watchdog.observers import Observer # type: ignore - -# patch encoding / decoding of paths in watchdog - - -def _patched_decode(path: Union[str, bytes]) -> str: - if isinstance(path, bytes): - return os.fsdecode(path) - return path - - -def _patched_encode(path: Union[str, bytes]) -> bytes: - if isinstance(path, str): - return os.fsencode(path) - return path - - -try: - from watchdog.utils import unicode_paths -except ImportError: - pass -else: - unicode_paths.decode = _patched_decode - unicode_paths.encode = _patched_encode - - __all__ = ["Observer"] diff --git a/src/maestral/fsevents/fsevents.py b/src/maestral/fsevents/fsevents.py deleted file mode 100644 index f9d98924b..000000000 --- a/src/maestral/fsevents/fsevents.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright 2011 Yesudeep Mangalapilly -# Copyright 2012 Google, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from watchdog.observers.fsevents import ( # type: ignore - FSEventsEmitter, - FSEventsObserver, - FileDeletedEvent, - FileModifiedEvent, - FileMovedEvent, - FileCreatedEvent, - DirDeletedEvent, - DirModifiedEvent, - DirMovedEvent, - DirCreatedEvent, - DEFAULT_OBSERVER_TIMEOUT, - BaseObserver, -) -from watchdog.utils.dirsnapshot import DirectorySnapshot - - -class OrderedFSEventsEmitter(FSEventsEmitter): - """Ordered file system event emitter for macOS - - This subclasses FSEventsEmitter to guarantee an order of events which can be applied - to reproduce the new state from the old state. - """ - - def queue_events(self, timeout): - with self._lock: - if not self.watch.is_recursive and self.watch.path not in self.pathnames: - return - new_snapshot = DirectorySnapshot(self.watch.path, self.watch.is_recursive) - diff = new_snapshot - self.snapshot - - # add metadata modified events which will be missed by regular diff - try: - ctime_files_modified = set() - - for path in self.snapshot.paths & new_snapshot.paths: - if not self.snapshot.isdir(path): - if self.snapshot.inode(path) == new_snapshot.inode(path): - if ( - self.snapshot.stat_info(path).st_ctime - != new_snapshot.stat_info(path).st_ctime - ): - ctime_files_modified.add(path) - - files_modified = set(ctime_files_modified) | set(diff.files_modified) - except Exception as exc: - print(exc) - - # replace cached snapshot - self.snapshot = new_snapshot - - # Files. - for src_path in diff.files_deleted: - self.queue_event(FileDeletedEvent(src_path)) - for src_path in files_modified: - self.queue_event(FileModifiedEvent(src_path)) - for src_path, dest_path in diff.files_moved: - self.queue_event(FileMovedEvent(src_path, dest_path)) - for src_path in diff.files_created: - self.queue_event(FileCreatedEvent(src_path)) - - # Directories. - for src_path in diff.dirs_deleted: - self.queue_event(DirDeletedEvent(src_path)) - for src_path in diff.dirs_modified: - self.queue_event(DirModifiedEvent(src_path)) - for src_path, dest_path in diff.dirs_moved: - self.queue_event(DirMovedEvent(src_path, dest_path)) - for src_path in diff.dirs_created: - self.queue_event(DirCreatedEvent(src_path)) - - # free some memory - del diff - del files_modified - - -class OrderedFSEventsObserver(FSEventsObserver): - def __init__(self, timeout=DEFAULT_OBSERVER_TIMEOUT): - BaseObserver.__init__( - self, emitter_class=OrderedFSEventsEmitter, timeout=timeout - ) diff --git a/src/maestral/fsevents/polling.py b/src/maestral/fsevents/polling.py index 3577714e0..3768674bf 100644 --- a/src/maestral/fsevents/polling.py +++ b/src/maestral/fsevents/polling.py @@ -1,6 +1,50 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -# +""" +Looking at the source code for :class:`watchdog.utils.dirsnapshot.DirectorySnapshotDiff`, +the event types are categorised as follows: + +* Created event: The inode is unique to the new snapshot. The path may be unique to the + new snapshot or exist in both. In the second case, there will be a preceding Deleted + event or a Moved event with the path as starting point (the old item was deleted or + moved away). + +* Deleted event: The inode is unique to the old snapshot. The path may be unique to the + old snapshot or exist in both. In the second case, there will be a subsequent Created + event or a Moved event with the path as end point (something else was created at or + moved to the location). + +* Moved event: The inode exists in both snapshots but with different paths. + +* Modified event: The inode exists in both snapshots and the mtime or file size are + different. DirectorySnapshotDiff will always use the inode’s path from the old + snapshot. + +From the above classification, there can be at most two created/deleted/moved events +that share the same path in one snapshot diff: + + * Deleted(path1) + Created(path1) + * Moved(path1, path2) + Created(path1) + * Deleted(path1) + Moved(path0, path1) + +Any Modified event will come before a Moved event or stand alone. Modified events will +never be combined by themselves with created or deleted events because they require the +inode to be present in both snapshots. + +From the above, we can achieve correct ordering for unique path by always adding Deleted +events to the queue first, Modified events second, Moved events third and Created events +last: + + Deleted -> Modified -> Moved -> Created + +The ordering won’t be correct between unrelated paths and between files and folder. The +first does not matter for syncing. We solve the second by assuming that when a directory +is deleted, so are its children. And before a child is created, its parent directory +must exist. + +MovedEvents which are not unique (their paths appear in other events) will be split +into Deleted and Created events by Maestral. +""" + # Copyright 2011 Yesudeep Mangalapilly # Copyright 2012 Google, Inc. # diff --git a/src/maestral/main.py b/src/maestral/main.py index a34acae56..8bde98e7d 100644 --- a/src/maestral/main.py +++ b/src/maestral/main.py @@ -1071,7 +1071,7 @@ def _remove_after_excluded(self, dbx_path: str) -> None: pass else: event_cls = DirDeletedEvent if osp.isdir(local_path) else FileDeletedEvent - with self.monitor.fs_event_handler.ignore(event_cls(local_path)): + with self.monitor.sync.fs_events.ignore(event_cls(local_path)): delete(local_path) def include_item(self, dbx_path: str) -> None: diff --git a/src/maestral/sync.py b/src/maestral/sync.py index 01dec8a70..876e2edd4 100644 --- a/src/maestral/sync.py +++ b/src/maestral/sync.py @@ -72,7 +72,6 @@ # local imports from . import notify from .config import MaestralConfig, MaestralState -from .fsevents import Observer from .constants import ( IDLE, SYNCING, @@ -114,6 +113,7 @@ ItemType, ChangeType, ) +from .fsevents import Observer from .utils import removeprefix, sanitize_string from .utils.caches import LRUCache from .utils.path import ( @@ -186,6 +186,12 @@ def __init__( self.ttl = ttl self.recursive = recursive + def __repr__(self) -> str: + return ( + f"<{self.__class__.__name__}(event={self.event}, " + f"recursive={self.recursive}, ttl={self.ttl})>" + ) + class FSEventHandler(FileSystemEventHandler): """A local file event handler @@ -231,6 +237,9 @@ def __init__( self.file_event_types = file_event_types self.dir_event_types = dir_event_types + self.file_event_types = file_event_types + self.dir_event_types = dir_event_types + self._ignored_events = [] self.ignore_timeout = 2.0 self.local_file_event_queue = Queue() @@ -369,10 +378,42 @@ def on_any_event(self, event: FileSystemEvent) -> None: if self._is_ignored(event): return + self.queue_event(event) + + def queue_event(self, event: FileSystemEvent) -> None: + """ + Queues an individual file system event. Notifies / wakes up all threads that are + waiting with :meth:`wait_for_event`. + + :param event: File system event to queue. + """ with self.has_events: self.local_file_event_queue.put(event) self.has_events.notify_all() + def wait_for_event(self, timeout: float = 40) -> bool: + """ + Blocks until an event is available in the queue or a timeout occurs, whichever + comes first. You can use with method to wait for file system events in another + thread. + + .. note:: If there are multiple threads waiting for events, all of them will be + notified. If one of those threads starts getting events from + :attr:`local_file_event_queue`, other threads may find that queue empty. You + should therefore always be prepared to handle an empty queue, if if this + method returns ``True``. + + :param timeout: Maximum time to block in seconds. + :returns: ``True`` if an event is available, ``False`` if the call returns due + to a timeout. + """ + + with self.has_events: + if self.local_file_event_queue.qsize() > 0: + return True + self.has_events.wait(timeout) + return self.local_file_event_queue.qsize() > 0 + class PersistentStateMutableSet(abc.MutableSet): """Wraps a list in our state file as a MutableSet @@ -449,7 +490,6 @@ class SyncEngine: conflict resolution and updates to our index. :param client: Dropbox API client instance. - :param fs_events_handler: File system event handler to inform us of local events. """ sync_errors: Set[SyncError] @@ -459,11 +499,11 @@ class SyncEngine: _max_history = 1000 _num_threads = min(32, cpu_count * 3) - def __init__(self, client: DropboxClient, fs_events_handler: FSEventHandler): + def __init__(self, client: DropboxClient): self.client = client self.config_name = self.client.config_name - self.fs_events = fs_events_handler + self.fs_events = FSEventHandler() self.sync_lock = RLock() self._db_lock = RLock() @@ -1560,8 +1600,8 @@ def _get_local_changes_while_inactive(self) -> Tuple[List[FileSystemEvent], floa elif is_modified: if snapshot.isdir(path) and index_entry.is_directory: # type: ignore - event = DirModifiedEvent(path) - changes.append(event) + # We don't emit `DirModifiedEvent`s. + pass elif not snapshot.isdir(path) and not index_entry.is_directory: # type: ignore event = FileModifiedEvent(path) changes.append(event) @@ -1604,13 +1644,7 @@ def wait_for_local_changes(self, timeout: float = 40) -> bool: logger.debug("Waiting for local changes since cursor: %s", self.local_cursor) - if self.fs_events.local_file_event_queue.qsize() > 0: - return True - - with self.fs_events.has_events: - self.fs_events.has_events.wait(timeout) - - return self.fs_events.local_file_event_queue.qsize() > 0 + return self.fs_events.wait_for_event(timeout) def upload_sync_cycle(self): """ @@ -2139,6 +2173,9 @@ def _on_local_moved(self, event: SyncEvent) -> Optional[Metadata]: :raises MaestralApiError: For any issues when syncing the item. """ + if event.local_path_from == self.dropbox_path: + self.ensure_dropbox_folder_present() + # fail fast on badly decoded paths validate_encoding(event.local_path) @@ -2395,6 +2432,9 @@ def _on_local_deleted(self, event: SyncEvent) -> Optional[Metadata]: :raises MaestralApiError: For any issues when syncing the item. """ + if event.local_path == self.dropbox_path: + self.ensure_dropbox_folder_present() + if self.is_excluded_by_user(event.dbx_path): logger.debug( 'Not deleting "%s": is excluded by selective sync', event.dbx_path @@ -2627,6 +2667,7 @@ def list_remote_changes_iterator( # Pick up where we left off. This may be an interrupted indexing / # pagination through changes or a completely new set of changes. logger.info("Fetching remote changes...") + logger.debug("Fetching remote changes since cursor: %s", last_cursor) changes_iter = self.client.list_remote_changes_iterator(last_cursor) for changes in changes_iter: @@ -3168,10 +3209,7 @@ def _on_remote_file(self, event: SyncEvent) -> Optional[SyncEvent]: else: preserve_permissions = False - ignore_events = [ - FileMovedEvent(tmp_fname, local_path), - FileCreatedEvent(local_path), # sometimes emitted on macOS - ] + ignore_events = [FileMovedEvent(tmp_fname, local_path)] if preserve_permissions: # ignore FileModifiedEvent when changing permissions @@ -3341,7 +3379,7 @@ def rescan(self, local_path: str) -> None: logger.debug('Rescanning "%s"', local_path) if osp.isfile(local_path): - self.fs_events.local_file_event_queue.put(FileModifiedEvent(local_path)) + self.fs_events.queue_event(FileModifiedEvent(local_path)) elif osp.isdir(local_path): # add created and deleted events of children as appropriate @@ -3352,9 +3390,9 @@ def rescan(self, local_path: str) -> None: for path in snapshot.paths: if snapshot.isdir(path): - self.fs_events.local_file_event_queue.put(DirCreatedEvent(path)) + self.fs_events.queue_event(DirCreatedEvent(path)) else: - self.fs_events.local_file_event_queue.put(FileModifiedEvent(path)) + self.fs_events.queue_event(FileModifiedEvent(path)) # add deleted events @@ -3365,22 +3403,16 @@ def rescan(self, local_path: str) -> None: .all() ) + dbx_root_lower = self.dropbox_path.lower() + for entry in entries: - child_path_uncased = ( - f"{self.dropbox_path}{entry.dbx_path_lower}".lower() - ) + child_path_uncased = f"{dbx_root_lower}{entry.dbx_path_lower}" if child_path_uncased not in lowercase_snapshot_paths: - local_child_path = self.to_local_path_from_cased( - entry.dbx_path_cased - ) + local_child = self.to_local_path_from_cased(entry.dbx_path_cased) if entry.is_directory: - self.fs_events.local_file_event_queue.put( - DirDeletedEvent(local_child_path) - ) + self.fs_events.queue_event(DirDeletedEvent(local_child)) else: - self.fs_events.local_file_event_queue.put( - FileDeletedEvent(local_child_path) - ) + self.fs_events.queue_event(FileDeletedEvent(local_child)) elif not osp.exists(local_path): dbx_path = self.to_dbx_path(local_path) @@ -3389,16 +3421,9 @@ def rescan(self, local_path: str) -> None: if local_entry: if local_entry.is_directory: - self.fs_events.local_file_event_queue.put( - DirDeletedEvent(local_path) - ) + self.fs_events.queue_event(DirDeletedEvent(local_path)) else: - self.fs_events.local_file_event_queue.put( - FileDeletedEvent(local_path) - ) - - with self.fs_events.has_events: - self.fs_events.has_events.notify_all() + self.fs_events.queue_event(FileDeletedEvent(local_path)) def _clean_history(self): """Commits new events and removes all events older than ``_keep_history`` from @@ -3661,8 +3686,7 @@ def __init__(self, client: DropboxClient): self.added_item_queue = Queue() - self.fs_event_handler = FSEventHandler() - self.sync = SyncEngine(self.client, self.fs_event_handler) + self.sync = SyncEngine(self.client) self._startup_time = -1.0 @@ -3735,7 +3759,7 @@ def start(self) -> None: self.local_observer_thread = Observer(timeout=40) self.local_observer_thread.setName("maestral-fsobserver") self._watch = self.local_observer_thread.schedule( - self.fs_event_handler, self.sync.dropbox_path, recursive=True + self.sync.fs_events, self.sync.dropbox_path, recursive=True ) for i, emitter in enumerate(self.local_observer_thread.emitters): emitter.setName(f"maestral-fsemitter-{i}") @@ -3801,7 +3825,7 @@ def start(self) -> None: self.running.set() self.autostart.set() - self.fs_event_handler.enable() + self.sync.fs_events.enable() self.startup_thread.start() self.upload_thread.start() self.download_thread.start() @@ -3818,7 +3842,7 @@ def stop(self) -> None: logger.info("Shutting down threads...") - self.fs_event_handler.disable() + self.sync.fs_events.disable() self.running.clear() self.startup_completed.clear() self.autostart.clear() diff --git a/tests/linked/conftest.py b/tests/linked/conftest.py index 6cb3bf8d6..281621828 100644 --- a/tests/linked/conftest.py +++ b/tests/linked/conftest.py @@ -27,6 +27,8 @@ resources = os.path.dirname(__file__) + "/resources" +fsevents_logger = logging.getLogger("fsevents") +fsevents_logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG) @@ -130,7 +132,7 @@ def proxy(m): # helper functions -def wait_for_idle(m: Maestral, minimum: int = 4): +def wait_for_idle(m: Maestral, minimum: int = 5): """Blocks until Maestral instance is idle for at least `minimum` sec.""" t0 = time.time() diff --git a/tests/linked/test_sync.py b/tests/linked/test_sync.py index 9ef7a19bd..eaa013972 100644 --- a/tests/linked/test_sync.py +++ b/tests/linked/test_sync.py @@ -191,7 +191,7 @@ def test_local_deletion_during_upload(m): # we mimic a deletion during upload by queueing a fake FileCreatedEvent fake_created_event = FileCreatedEvent(m.test_folder_local + "/file.txt") - m.monitor.fs_event_handler.local_file_event_queue.put(fake_created_event) + m.monitor.sync.fs_events.queue_event(fake_created_event) wait_for_idle(m) @@ -238,8 +238,9 @@ def test_rapid_remote_changes(m): mode=WriteMode.update(md.rev), ) + # reset file content with open(resources + "/file.txt", "w") as f: - f.write("content") # reset file content + f.write("content") wait_for_idle(m) @@ -295,7 +296,7 @@ def test_folder_tree_created_remote(m): # test deleting remote tree m.client.remove("/sync_tests/nested_folder") - wait_for_idle(m, 10) + wait_for_idle(m, 15) assert_synced(m) assert_child_count(m, "/sync_tests", 0) @@ -310,14 +311,12 @@ def test_remote_file_replaced_by_folder(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace remote file with folder - m.client.remove("/sync_tests/file.txt") - m.client.make_dir("/sync_tests/file.txt") + # replace remote file with folder + m.client.remove("/sync_tests/file.txt") + m.client.make_dir("/sync_tests/file.txt") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -337,18 +336,16 @@ def test_remote_file_replaced_by_folder_and_unsynced_local_changes(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace remote file with folder - m.client.remove("/sync_tests/file.txt") - m.client.make_dir("/sync_tests/file.txt") + # replace remote file with folder + m.client.remove("/sync_tests/file.txt") + m.client.make_dir("/sync_tests/file.txt") - # create local changes - with open(m.test_folder_local + "/file.txt", "a") as f: - f.write(" modified") + # create local changes + with open(m.test_folder_local + "/file.txt", "a") as f: + f.write(" modified") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -363,17 +360,15 @@ def test_remote_file_replaced_by_folder_and_unsynced_local_changes(m): def test_remote_folder_replaced_by_file(m): """Tests the download sync when a folder is replaced by a file.""" - os.mkdir(m.test_folder_local + "/folder") - wait_for_idle(m) - - m.stop_sync() + m.client.make_dir("/sync_tests/folder") wait_for_idle(m) # replace remote folder with file - m.client.remove("/sync_tests/folder") - m.client.upload(resources + "/file.txt", "/sync_tests/folder") - m.start_sync() + with m.sync.sync_lock: + m.client.remove("/sync_tests/folder") + m.client.upload(resources + "/file.txt", "/sync_tests/folder") + wait_for_idle(m) assert_synced(m) @@ -393,17 +388,15 @@ def test_remote_folder_replaced_by_file_and_unsynced_local_changes(m): os.mkdir(m.test_folder_local + "/folder") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace remote folder with file - m.client.remove("/sync_tests/folder") - m.client.upload(resources + "/file.txt", "/sync_tests/folder") + # replace remote folder with file + m.client.remove("/sync_tests/folder") + m.client.upload(resources + "/file.txt", "/sync_tests/folder") - # create local changes - os.mkdir(m.test_folder_local + "/folder/subfolder") + # create local changes + os.mkdir(m.test_folder_local + "/folder/subfolder") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -421,13 +414,12 @@ def test_local_folder_replaced_by_file(m): os.mkdir(m.test_folder_local + "/folder") wait_for_idle(m) - m.stop_sync() + with m.sync.sync_lock: - # replace local folder with file - delete(m.test_folder_local + "/folder") - shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") + # replace local folder with file + delete(m.test_folder_local + "/folder") + shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -449,17 +441,15 @@ def test_local_folder_replaced_by_file_and_unsynced_remote_changes(m): os.mkdir(m.test_folder_local + "/folder") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace local folder with file - delete(m.test_folder_local + "/folder") - shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") + # replace local folder with file + delete(m.test_folder_local + "/folder") + shutil.copy(resources + "/file.txt", m.test_folder_local + "/folder") - # create remote changes - m.client.upload(resources + "/file1.txt", "/sync_tests/folder/file.txt") + # create remote changes + m.client.upload(resources + "/file1.txt", "/sync_tests/folder/file.txt") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -476,14 +466,12 @@ def test_local_file_replaced_by_folder(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace local file with folder - os.unlink(m.test_folder_local + "/file.txt") - os.mkdir(m.test_folder_local + "/file.txt") + # replace local file with folder + os.unlink(m.test_folder_local + "/file.txt") + os.mkdir(m.test_folder_local + "/file.txt") - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -506,21 +494,19 @@ def test_local_file_replaced_by_folder_and_unsynced_remote_changes(m): shutil.copy(resources + "/file.txt", m.test_folder_local + "/file.txt") wait_for_idle(m) - m.stop_sync() - wait_for_idle(m) + with m.sync.sync_lock: - # replace local file with folder - os.unlink(m.test_folder_local + "/file.txt") - os.mkdir(m.test_folder_local + "/file.txt") + # replace local file with folder + os.unlink(m.test_folder_local + "/file.txt") + os.mkdir(m.test_folder_local + "/file.txt") - # create remote changes - m.client.upload( - resources + "/file1.txt", - "/sync_tests/file.txt", - mode=WriteMode.overwrite, - ) + # create remote changes + m.client.upload( + resources + "/file1.txt", + "/sync_tests/file.txt", + mode=WriteMode.overwrite, + ) - m.start_sync() wait_for_idle(m) assert_synced(m) @@ -672,14 +658,14 @@ def test_mignore(m): os.mkdir(m.test_folder_local + "/foo") wait_for_idle(m) - assert not (m.client.get_metadata("/sync_tests/foo")) + assert not m.client.get_metadata("/sync_tests/foo") # 3) test that renaming an item excludes it move(m.test_folder_local + "/bar", m.test_folder_local + "/build") wait_for_idle(m) - assert not (m.client.get_metadata("/sync_tests/build")) + assert not m.client.get_metadata("/sync_tests/build") # 4) test that renaming an item includes it @@ -930,7 +916,7 @@ def generate_sync_events(): duration = timeit.timeit(stmt=generate_sync_events, setup=setup, number=n_loops) - assert duration < 3 # expected ~ 1.8 sec + assert duration < 4 # expected ~ 1.8 sec def test_invalid_pending_download(m): diff --git a/tests/offline/conftest.py b/tests/offline/conftest.py index a0c7059cb..dc1c8ca8e 100644 --- a/tests/offline/conftest.py +++ b/tests/offline/conftest.py @@ -7,7 +7,7 @@ import pytest from maestral.main import Maestral, logger -from maestral.sync import SyncEngine, Observer, FSEventHandler +from maestral.sync import SyncEngine, Observer from maestral.client import DropboxClient from maestral.config import list_configs, remove_configuration from maestral.daemon import stop_maestral_daemon_process, Stop @@ -32,10 +32,8 @@ def sync(): local_dir = osp.join(get_home_dir(), "dummy_dir") os.mkdir(local_dir) - fs_events_handler = FSEventHandler() - fs_events_handler.enable() - - sync = SyncEngine(DropboxClient("test-config"), fs_events_handler) + sync = SyncEngine(DropboxClient("test-config")) + sync.fs_events.enable() sync.dropbox_path = local_dir observer = Observer() diff --git a/tests/offline/test_cleaning_events.py b/tests/offline/test_cleaning_events.py index fe3ba97e0..5b608115a 100644 --- a/tests/offline/test_cleaning_events.py +++ b/tests/offline/test_cleaning_events.py @@ -20,7 +20,7 @@ @pytest.fixture def sync(): - sync = SyncEngine(DropboxClient("test-config"), None) + sync = SyncEngine(DropboxClient("test-config")) sync.dropbox_path = "/" yield sync