From 8515e14307401e317726245f575c0ceecb91c865 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 7 Dec 2020 14:08:45 +0100 Subject: [PATCH 1/8] Remove spurious whitespace --- src/watchdog/observers/fsevents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/watchdog/observers/fsevents.py b/src/watchdog/observers/fsevents.py index 26ae8becd..794a967c3 100644 --- a/src/watchdog/observers/fsevents.py +++ b/src/watchdog/observers/fsevents.py @@ -111,7 +111,7 @@ def queue_events(self, timeout): self.queue_event(DirModifiedEvent(os.path.dirname(event.path))) # TODO: generate events for tree - elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod : + elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod: cls = DirModifiedEvent if event.is_directory else FileModifiedEvent self.queue_event(cls(event.path)) From 43989bd31997f077ffb1b98dbffbb9ebe6579db7 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 7 Dec 2020 14:13:50 +0100 Subject: [PATCH 2/8] Expose missing fsevents properties --- src/watchdog_fsevents.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/watchdog_fsevents.c b/src/watchdog_fsevents.c index bf86a9c6d..726be6888 100644 --- a/src/watchdog_fsevents.c +++ b/src/watchdog_fsevents.c @@ -149,11 +149,29 @@ PyObject* NativeEventTypeID(PyObject* instance, void* closure) Py_RETURN_FALSE; \ } +FLAG_PROPERTY(IsMustScanSubDirs, kFSEventStreamEventFlagMustScanSubDirs) +FLAG_PROPERTY(IsUserDropped, kFSEventStreamEventFlagUserDropped) +FLAG_PROPERTY(IsKernelDropped, kFSEventStreamEventFlagKernelDropped) +FLAG_PROPERTY(IsEventIdsWrapped, kFSEventStreamEventFlagEventIdsWrapped) +FLAG_PROPERTY(IsHistoryDone, kFSEventStreamEventFlagHistoryDone) +FLAG_PROPERTY(IsRootChanged, kFSEventStreamEventFlagRootChanged) +FLAG_PROPERTY(IsMount, kFSEventStreamEventFlagMount) +FLAG_PROPERTY(IsUnmount, kFSEventStreamEventFlagUnmount) FLAG_PROPERTY(IsCreated, kFSEventStreamEventFlagItemCreated) FLAG_PROPERTY(IsRemoved, kFSEventStreamEventFlagItemRemoved) +FLAG_PROPERTY(IsInodeMetaMod, kFSEventStreamEventFlagItemInodeMetaMod) FLAG_PROPERTY(IsRenamed, kFSEventStreamEventFlagItemRenamed) FLAG_PROPERTY(IsModified, kFSEventStreamEventFlagItemModified) +FLAG_PROPERTY(IsItemFinderInfoMod, kFSEventStreamEventFlagItemFinderInfoMod) +FLAG_PROPERTY(IsChangeOwner, kFSEventStreamEventFlagItemChangeOwner) +FLAG_PROPERTY(IsXattrMod, kFSEventStreamEventFlagItemXattrMod) +FLAG_PROPERTY(IsFile, kFSEventStreamEventFlagItemIsFile) FLAG_PROPERTY(IsDirectory, kFSEventStreamEventFlagItemIsDir) +FLAG_PROPERTY(IsSymlink, kFSEventStreamEventFlagItemIsSymlink) +FLAG_PROPERTY(IsOwnEvent, kFSEventStreamEventFlagOwnEvent) +FLAG_PROPERTY(IsHardlink, kFSEventStreamEventFlagItemIsHardlink) +FLAG_PROPERTY(IsLastHardlink, kFSEventStreamEventFlagItemIsLastHardlink) +FLAG_PROPERTY(IsCloned, kFSEventStreamEventFlagItemCloned) static int NativeEventInit(NativeEventObject *self, PyObject *args, PyObject *kwds) { @@ -171,11 +189,29 @@ static PyGetSetDef NativeEventProperties[] = { {"flags", NativeEventTypeFlags, NULL, "The raw mask of flags as returend by FSEvents", NULL}, {"path", NativeEventTypePath, NULL, "The path for which this event was generated", NULL}, {"id", NativeEventTypeID, NULL, "The id of the generated event", NULL}, + {"must_scan_subdirs", NativeEventTypeIsMustScanSubDirs, NULL, "True if application must rescan all subdirectories", NULL}, + {"is_user_dropped", NativeEventTypeIsUserDropped, NULL, "True if a failure during event buffering occured", NULL}, + {"is_kernel_dropped", NativeEventTypeIsKernelDropped, NULL, "True if a failure during event buffering occured", NULL}, + {"is_event_ids_wrapped", NativeEventTypeIsEventIdsWrapped, NULL, "True if event_id wrapped around", NULL}, + {"is_history_done", NativeEventTypeIsHistoryDone, NULL, "True if all historical events are done", NULL}, + {"is_root_changed", NativeEventTypeIsRootChanged, NULL, "True if a change to one of the directories along the path to one of the directories you watch occurred", NULL}, + {"is_mount", NativeEventTypeIsMount, NULL, "True if a volume is mounted underneath one of the paths being monitored", NULL}, + {"is_unmount", NativeEventTypeIsUnmount, NULL, "True if a volume is unmounted underneath one of the paths being monitored", NULL}, {"is_created", NativeEventTypeIsCreated, NULL, "True if self.path was created on the filesystem", NULL}, {"is_removed", NativeEventTypeIsRemoved, NULL, "True if self.path was removed from the filesystem", NULL}, + {"is_inode_meta_mod", NativeEventTypeIsInodeMetaMod, NULL, "True if meta data for self.path was modified ", NULL}, {"is_renamed", NativeEventTypeIsRenamed, NULL, "True if self.path was renamed on the filesystem", NULL}, {"is_modified", NativeEventTypeIsModified, NULL, "True if self.path was modified", NULL}, + {"is_item_finder_info_modified", NativeEventTypeIsItemFinderInfoMod, NULL, "True if FinderInfo for self.path was modified", NULL}, + {"is_owner_change", NativeEventTypeIsChangeOwner, NULL, "True if self.path had its ownership changed", NULL}, + {"is_xattr_mod", NativeEventTypeIsXattrMod, NULL, "True if extended attributes for self.path were modified ", NULL}, + {"is_file", NativeEventTypeIsFile, NULL, "True if self.path is a file", NULL}, {"is_directory", NativeEventTypeIsDirectory, NULL, "True if self.path is a directory", NULL}, + {"is_symlink", NativeEventTypeIsSymlink, NULL, "True if self.path is a symbolic link", NULL}, + {"is_own_event", NativeEventTypeIsOwnEvent, NULL, "True if the event originated from our own process", NULL}, + {"is_hardlink", NativeEventTypeIsHardlink, NULL, "True if self.path is a hard link", NULL}, + {"is_last_hardlink", NativeEventTypeIsLastHardlink, NULL, "True if self.path was the last hard link", NULL}, + {"is_cloned", NativeEventTypeIsCloned, NULL, "True if self.path is a clone or was cloned", NULL}, {NULL, NULL, NULL, NULL, NULL}, }; From e5508cec52b3c23f6d9caab936c9b4b9d7360df6 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 7 Dec 2020 14:19:44 +0100 Subject: [PATCH 3/8] Use PyCapsule with Python 2.7 Addresses one possible source of memory corruption, and simplifies the code. --- src/watchdog_fsevents.c | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/watchdog_fsevents.c b/src/watchdog_fsevents.c index 726be6888..674e036e1 100644 --- a/src/watchdog_fsevents.c +++ b/src/watchdog_fsevents.c @@ -243,9 +243,8 @@ PyObject *watch_to_stream = NULL; /** - * PyCapsule destructor for Python 3 compatibility + * PyCapsule destructor */ -#if PY_MAJOR_VERSION >= 3 static void watchdog_pycapsule_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, NULL); @@ -253,7 +252,6 @@ static void watchdog_pycapsule_destructor(PyObject *ptr) PyMem_Free(p); } } -#endif /** @@ -515,11 +513,7 @@ watchdog_add_watch(PyObject *self, PyObject *args) stream_ref = watchdog_FSEventStreamCreate(stream_callback_info_ref, paths_to_watch, (FSEventStreamCallback) &watchdog_FSEventStreamCallback); -#if PY_MAJOR_VERSION >= 3 value = PyCapsule_New(stream_ref, NULL, watchdog_pycapsule_destructor); -#else - value = PyCObject_FromVoidPtr(stream_ref, PyMem_Free); -#endif PyDict_SetItem(watch_to_stream, watch, value); /* Get a reference to the runloop for the emitter thread @@ -531,11 +525,7 @@ watchdog_add_watch(PyObject *self, PyObject *args) } else { -#if PY_MAJOR_VERSION >= 3 run_loop_ref = PyCapsule_GetPointer(value, NULL); -#else - run_loop_ref = PyCObject_AsVoidPtr(value); -#endif } /* Schedule the stream with the obtained runloop. */ @@ -586,11 +576,7 @@ watchdog_read_events(PyObject *self, PyObject *args) if (G_IS_NULL(value)) { run_loop_ref = CFRunLoopGetCurrent(); -#if PY_MAJOR_VERSION >= 3 value = PyCapsule_New(run_loop_ref, NULL, watchdog_pycapsule_destructor); -#else - value = PyCObject_FromVoidPtr(run_loop_ref, PyMem_Free); -#endif PyDict_SetItem(thread_to_run_loop, emitter_thread, value); Py_INCREF(emitter_thread); Py_INCREF(value); @@ -626,11 +612,7 @@ watchdog_remove_watch(PyObject *self, PyObject *watch) PyObject *value = PyDict_GetItem(watch_to_stream, watch); PyDict_DelItem(watch_to_stream, watch); -#if PY_MAJOR_VERSION >= 3 FSEventStreamRef stream_ref = PyCapsule_GetPointer(value, NULL); -#else - FSEventStreamRef stream_ref = PyCObject_AsVoidPtr(value); -#endif FSEventStreamStop(stream_ref); FSEventStreamInvalidate(stream_ref); @@ -654,11 +636,7 @@ watchdog_stop(PyObject *self, PyObject *emitter_thread) goto success; } -#if PY_MAJOR_VERSION >= 3 CFRunLoopRef run_loop_ref = PyCapsule_GetPointer(value, NULL); -#else - CFRunLoopRef run_loop_ref = PyCObject_AsVoidPtr(value); -#endif G_RETURN_NULL_IF(PyErr_Occurred()); /* Stop the run loop. */ From d340e35e88734aa0090dc563baa73d4155e948d8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 7 Dec 2020 14:22:38 +0100 Subject: [PATCH 4/8] Fix event_id construction --- src/watchdog_fsevents.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/watchdog_fsevents.c b/src/watchdog_fsevents.c index 674e036e1..e48828a5f 100644 --- a/src/watchdog_fsevents.c +++ b/src/watchdog_fsevents.c @@ -313,7 +313,7 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef stream_ref, } for (i = 0; i < num_events; ++i) { - id = PyLong_FromLongLong(event_flags[i]); + id = PyLong_FromLongLong(event_ids[i]); #if PY_MAJOR_VERSION >= 3 path = PyUnicode_FromString(event_paths[i]); flags = PyLong_FromLong(event_flags[i]); From 514271399f8d3776431a51d0cd472585121e47be Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 7 Dec 2020 15:43:48 +0100 Subject: [PATCH 5/8] Ensure UTF-8 encoded paths are used --- src/watchdog_fsevents.c | 59 ++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/src/watchdog_fsevents.c b/src/watchdog_fsevents.c index e48828a5f..bcd5e54b6 100644 --- a/src/watchdog_fsevents.c +++ b/src/watchdog_fsevents.c @@ -357,6 +357,48 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef stream_ref, PyGILState_Release(gil_state); } +/** + * Converts a Python string object to an UTF-8 encoded ``CFStringRef``. + * + * :param py_string: + * A Python unicode or utf-8 encoded bytestring object. + * :returns: + * A new ``CFStringRef`` with the contents of ``py_string``, or ``NULL`` if an error occurred. + */ +CFStringRef PyString_AsUTF8EncodedCFStringRef(PyObject *py_string) +{ + CFStringRef cf_string = NULL; + const char *c_string = NULL; + PyObject *helper = NULL; + + if (PyUnicode_Check(py_string)) { + helper = PyUnicode_AsUTF8String(py_string); + } else if (PyBytes_Check(py_string)) { + PyObject *utf8 = PyUnicode_FromEncodedObject(py_string, NULL, "strict"); + if (!utf8) { + return NULL; + } + Py_DECREF(utf8); + helper = PyObject_Bytes(py_string); + } else { + PyErr_SetString(PyExc_TypeError, "Path to watch must be a string or a UTF-8 encoded bytes object."); + return NULL; + } + + if (!helper) + return NULL; + + c_string = PyBytes_AsString(helper); + if (c_string) { + cf_string = CFStringCreateWithCString(kCFAllocatorDefault, c_string,kCFStringEncodingUTF8); + Py_DECREF(c_string); + } + + Py_XDECREF(helper); + + return cf_string; +} + /** * Converts a list of Python strings to a ``CFMutableArray`` of @@ -375,7 +417,6 @@ watchdog_CFMutableArrayRef_from_PyStringList(PyObject *py_string_list) { Py_ssize_t i = 0; Py_ssize_t string_list_size = 0; - const char *c_string = NULL; CFMutableArrayRef array_of_cf_string = NULL; CFStringRef cf_string = NULL; PyObject *py_string = NULL; @@ -395,18 +436,10 @@ watchdog_CFMutableArrayRef_from_PyStringList(PyObject *py_string_list) { py_string = PyList_GetItem(py_string_list, i); G_RETURN_NULL_IF_NULL(py_string); -#if PY_MAJOR_VERSION >= 3 - if (PyUnicode_Check(py_string)) { - c_string = PyUnicode_AsUTF8(py_string); - } else { - c_string = PyBytes_AS_STRING(py_string); - } -#else - c_string = PyString_AS_STRING(py_string); -#endif - cf_string = CFStringCreateWithCString(kCFAllocatorDefault, - c_string, - kCFStringEncodingUTF8); + + cf_string = PyString_AsUTF8EncodedCFStringRef(py_string); + G_RETURN_NULL_IF_NULL(cf_string); + CFArraySetValueAtIndex(array_of_cf_string, i, cf_string); CFRelease(cf_string); } From 14bb4550f8c0cfd7a3a69ffcbe12999a127b4359 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 9 Dec 2020 14:27:03 +0100 Subject: [PATCH 6/8] Fix regression introduced by Utf8 conversion func Python 2.7 didn't like the double string conversion. Also improve some of the error handling code to behave better, as well as add error handling to th PyCapsule creation. --- src/watchdog_fsevents.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/watchdog_fsevents.c b/src/watchdog_fsevents.c index bcd5e54b6..75915122c 100644 --- a/src/watchdog_fsevents.c +++ b/src/watchdog_fsevents.c @@ -306,9 +306,9 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef stream_ref, py_event_ids = PyList_New(num_events); if (G_NOT(py_event_paths && py_event_flags && py_event_ids)) { - Py_DECREF(py_event_paths); - Py_DECREF(py_event_ids); - Py_DECREF(py_event_flags); + Py_XDECREF(py_event_paths); + Py_XDECREF(py_event_ids); + Py_XDECREF(py_event_flags); return /*NULL*/; } for (i = 0; i < num_events; ++i) @@ -368,34 +368,26 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef stream_ref, CFStringRef PyString_AsUTF8EncodedCFStringRef(PyObject *py_string) { CFStringRef cf_string = NULL; - const char *c_string = NULL; - PyObject *helper = NULL; if (PyUnicode_Check(py_string)) { - helper = PyUnicode_AsUTF8String(py_string); + PyObject* helper = PyUnicode_AsUTF8String(py_string); + if (!helper) { + return NULL; + } + cf_string = CFStringCreateWithCString(kCFAllocatorDefault, PyBytes_AS_STRING(helper), kCFStringEncodingUTF8); + Py_DECREF(helper); } else if (PyBytes_Check(py_string)) { PyObject *utf8 = PyUnicode_FromEncodedObject(py_string, NULL, "strict"); if (!utf8) { return NULL; } Py_DECREF(utf8); - helper = PyObject_Bytes(py_string); + cf_string = CFStringCreateWithCString(kCFAllocatorDefault, PyBytes_AS_STRING(py_string), kCFStringEncodingUTF8); } else { PyErr_SetString(PyExc_TypeError, "Path to watch must be a string or a UTF-8 encoded bytes object."); return NULL; } - if (!helper) - return NULL; - - c_string = PyBytes_AsString(helper); - if (c_string) { - cf_string = CFStringCreateWithCString(kCFAllocatorDefault, c_string,kCFStringEncodingUTF8); - Py_DECREF(c_string); - } - - Py_XDECREF(helper); - return cf_string; } @@ -546,7 +538,18 @@ watchdog_add_watch(PyObject *self, PyObject *args) stream_ref = watchdog_FSEventStreamCreate(stream_callback_info_ref, paths_to_watch, (FSEventStreamCallback) &watchdog_FSEventStreamCallback); + if (!stream_ref) { + PyMem_Del(stream_callback_info_ref); + PyErr_SetString(PyExc_RuntimeError, "Failed creating fsevent stream"); + return NULL; + } value = PyCapsule_New(stream_ref, NULL, watchdog_pycapsule_destructor); + if (!value || !PyCapsule_IsValid(value, NULL)) { + PyMem_Del(stream_callback_info_ref); + FSEventStreamInvalidate(stream_ref); + FSEventStreamRelease(stream_ref); + return NULL; + } PyDict_SetItem(watch_to_stream, watch, value); /* Get a reference to the runloop for the emitter thread From d961198ef142753942a9a4c7c869dbfcfc8252a9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 9 Dec 2020 14:30:11 +0100 Subject: [PATCH 7/8] Add test for recursive watch Inspired by issue #706 --- tests/test_fsevents.py | 47 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/tests/test_fsevents.py b/tests/test_fsevents.py index 27fa6dd87..068d6b86a 100644 --- a/tests/test_fsevents.py +++ b/tests/test_fsevents.py @@ -12,12 +12,13 @@ from functools import partial from os import mkdir, rmdir +from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer from watchdog.observers.api import ObservedWatch from watchdog.observers.fsevents import FSEventsEmitter from . import Queue -from .shell import mkdtemp, rm +from .shell import mkdtemp, rm, touch logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) @@ -96,7 +97,49 @@ def on_thread_stop(self): """ a = p("a") mkdir(a) - w = observer.schedule(event_queue, a, recursive=False) + w = observer.schedule(FileSystemEventHandler(), a, recursive=False) rmdir(a) time.sleep(0.1) observer.unschedule(w) + + +def test_watchdog_recursive(): + """ See https://github.com/gorakhargosh/watchdog/issues/706 + """ + from watchdog.observers import Observer + from watchdog.events import FileSystemEventHandler + import os.path + + class Handler(FileSystemEventHandler): + def __init__(self): + FileSystemEventHandler.__init__(self) + self.changes = [] + + def on_any_event(self, event): + self.changes.append(os.path.basename(event.src_path)) + + handler = Handler() + observer = Observer() + + watches = [] + watches.append(observer.schedule(handler, str(p('')), recursive=True)) + + try: + observer.start() + time.sleep(0.1) + + touch(p('my0.txt')) + mkdir(p('dir_rec')) + touch(p('dir_rec', 'my1.txt')) + + expected = {"dir_rec", "my0.txt", "my1.txt"} + timeout_at = time.time() + 5 + while not expected.issubset(handler.changes) and time.time() < timeout_at: + time.sleep(0.2) + + assert expected.issubset(handler.changes), "Did not find expected changes. Found: {}".format(handler.changes) + finally: + for watch in watches: + observer.unschedule(watch) + observer.stop() + observer.join(1) From 7be1a4df77a8a58a36c1ea98b76181dfac836f5c Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 9 Dec 2020 14:35:57 +0100 Subject: [PATCH 8/8] Limit tox to Python versions supported in this branch --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0da299c24..3c4f455b7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{310,39,38,37,36,35,27,py3,py} +envlist = py{35,27,py} skip_missing_interpreters = True [testenv]