Forked from serge-kilimoff/watchdog_persistent_observer.py
Last active
August 11, 2022 16:48
-
-
Save balloy/20ab6ed21363ddaf4e7b94a6448605fc to your computer and use it in GitHub Desktop.
Persistent watchdog observer. When watchdog re-start, check if new/modify/delete/etc.. files or directories since the last launch, and send events for suscribers handlers.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
# -*- coding: utf-8 -*- | |
""" | |
This file is forked from https://gist.github.com/serge-kilimoff/8163233 | |
Supports Python 3 and fixed a few spell errors. | |
""" | |
""" | |
Subclassing Observer for saving states of folders, and load this states at the next observation. | |
TODO : mapping events and handlers dispatching, for a shorter code. | |
""" | |
import pickle | |
import os | |
import time | |
from watchdog.observers import Observer | |
from watchdog.utils.dirsnapshot import DirectorySnapshot, DirectorySnapshotDiff | |
from watchdog.events import FileCreatedEvent, FileDeletedEvent, FileModifiedEvent, FileMovedEvent | |
from watchdog.events import DirCreatedEvent, DirDeletedEvent, DirModifiedEvent, DirMovedEvent | |
__author__ = "Serge Kilimoff-Goriatchkine" | |
__licence__ = 'MIT Licence' | |
class _EmptySnapshot(object): | |
@property | |
def stat_snapshot(self): | |
return dict() | |
@property | |
def paths(self): | |
return set() | |
def path(self, id): | |
return '' | |
class PersistentObserver(Observer): | |
def __init__(self, *args, **kwargs): | |
""" | |
Check if watching folders has changed since last observation. | |
If change detected, emit corresponding events at suscribers handlers. | |
At the `Observer.stop`, save states of folders with pickle for the next observation. | |
PARAMETERS | |
========== | |
save_to : unicode | |
path where save pickle dumping | |
protocol (optionnal): int | |
protocol used for dump current states of watching folders | |
""" | |
self._filename = kwargs.pop('save_to') | |
self._protocol = kwargs.pop('protocol', 0) | |
Observer.__init__(self, *args, **kwargs) | |
def start(self, *args, **kwargs): | |
previous_snapshots = dict() | |
if os.path.exists(self._filename): | |
with open(self._filename, 'rb') as f: | |
previous_snapshots = pickle.load(f) | |
for watcher, handlers in self._handlers.items(): | |
path = watcher.path | |
curr_snap = DirectorySnapshot(path) | |
pre_snap = previous_snapshots.get(path, _EmptySnapshot()) | |
diff = DirectorySnapshotDiff(pre_snap, curr_snap) | |
for handler in handlers: | |
# Dispatch files modifications | |
for new_path in diff.files_created: | |
handler.dispatch(FileCreatedEvent(new_path)) | |
for del_path in diff.files_deleted: | |
handler.dispatch(FileDeletedEvent(del_path)) | |
for mod_path in diff.files_modified: | |
handler.dispatch(FileModifiedEvent(mod_path)) | |
for mov_path in diff.files_moved: | |
handler.dispatch(FileMovedEvent(mov_path[0], mov_path[1])) | |
# Dispatch directories modifications | |
for new_dir in diff.dirs_created: | |
handler.dispatch(DirCreatedEvent(new_dir)) | |
for del_dir in diff.dirs_deleted: | |
handler.dispatch(DirDeletedEvent(del_dir)) | |
for mod_dir in diff.dirs_modified: | |
handler.dispatch(DirModifiedEvent(mod_dir)) | |
for mov_dir in diff.dirs_moved: | |
handler.dispatch(DirMovedEvent(mov_dir[0], mov_dir[1])) | |
Observer.start(self, *args, **kwargs) | |
def stop(self, *args, **kwargs): | |
snapshots = {handler.path : DirectorySnapshot(handler.path) for handler in self._handlers.keys()} | |
with open(self._filename, 'wb') as f: | |
pickle.dump(snapshots, f, self._protocol) | |
Observer.stop(self, *args, **kwargs) | |
if __name__ == "__main__": | |
# Simple example, derivated from watchdog doc. | |
import logging | |
from watchdog.events import LoggingEventHandler | |
logging.basicConfig(level=logging.DEBUG) | |
event_handler = LoggingEventHandler() | |
observer = PersistentObserver(save_to='/tmp/test.pickle') | |
observer.schedule(event_handler, path='/tmp/test', recursive=True) | |
observer.start() | |
try: | |
while True: | |
time.sleep(1) | |
except KeyboardInterrupt: | |
observer.stop() | |
observer.join() |
Manage to get it works by using dill
package
import dill as pickle
Hi @chunwai94,
Thanks for the info.
I didn't experience the same error -- maybe we're using different python/module versions. But it's good to know the workaround.
My versions:
Python 3.7.2
pickle 4.0
watchdog 0.9.0
Thanks.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
There is lambda function defined in DirectorySnapshot class, and Python can't pickle lambda functions.
_pickle.PicklingError: Can't pickle <function DirectorySnapshot.<lambda> at 0x10dff72f0>: attribute lookup DirectorySnapshot.<lambda> on watchdog.utils.dirsnapshot failed