diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 07c719f..3e0c99f 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -1,6 +1,6 @@ """ -This file contains definitions for a MEOW pattern based off of file events, +This file contains definitions for a MEOW pattern based off of file events, along with an appropriate monitor for said events. Author(s): David Marchant @@ -49,16 +49,16 @@ WATCHDOG_HASH = "file_hash" WATCHDOG_EVENT_KEYS = { WATCHDOG_BASE: str, - WATCHDOG_HASH: str, + WATCHDOG_HASH: str, **EVENT_KEYS } -def create_watchdog_event(path:str, rule:Any, base:str, time:float, +def create_watchdog_event(path:str, rule:Any, base:str, time:float, hash:str, extras:Dict[Any,Any]={})->Dict[Any,Any]: """Function to create a MEOW event dictionary.""" return create_event( - EVENT_TYPE_WATCHDOG, - path, + EVENT_TYPE_WATCHDOG, + path, rule, time, extras={ @@ -70,9 +70,6 @@ def create_watchdog_event(path:str, rule:Any, base:str, time:float, } ) -def valid_watchdog_event(event:Dict[str,Any])->None: - valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS) - class FileEventPattern(BasePattern): # The path at which events will trigger this pattern @@ -81,11 +78,11 @@ class FileEventPattern(BasePattern): triggering_file:str # Which types of event the pattern responds to event_mask:List[str] - def __init__(self, name:str, triggering_path:str, recipe:str, - triggering_file:str, event_mask:List[str]=_DEFAULT_MASK, - parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={}, + def __init__(self, name:str, triggering_path:str, recipe:str, + triggering_file:str, event_mask:List[str]=_DEFAULT_MASK, + parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}): - """FileEventPattern Constructor. This is used to match against file + """FileEventPattern Constructor. This is used to match against file system events, as caught by the python watchdog module.""" super().__init__(name, recipe, parameters, outputs, sweep) self._is_valid_triggering_path(triggering_path) @@ -96,70 +93,70 @@ class FileEventPattern(BasePattern): self.event_mask = event_mask def _is_valid_triggering_path(self, triggering_path:str)->None: - """Validation check for 'triggering_path' variable from main + """Validation check for 'triggering_path' variable from main constructor.""" valid_string( - triggering_path, - VALID_PATH_CHARS+'*', - min_length=1, + triggering_path, + VALID_PATH_CHARS+'*', + min_length=1, hint="FileEventPattern.triggering_path" ) if len(triggering_path) < 1: raise ValueError ( - f"triggiering path '{triggering_path}' is too short. " + f"triggering path '{triggering_path}' is too short. " "Minimum length is 1" ) def _is_valid_triggering_file(self, triggering_file:str)->None: - """Validation check for 'triggering_file' variable from main + """Validation check for 'triggering_file' variable from main constructor.""" valid_string( - triggering_file, + triggering_file, VALID_VARIABLE_NAME_CHARS, hint="FileEventPattern.triggering_file" ) def _is_valid_recipe(self, recipe:str)->None: - """Validation check for 'recipe' variable from main constructor. + """Validation check for 'recipe' variable from main constructor. Called within parent BasePattern constructor.""" valid_string( - recipe, + recipe, VALID_RECIPE_NAME_CHARS, hint="FileEventPattern.recipe" ) def _is_valid_parameters(self, parameters:Dict[str,Any])->None: - """Validation check for 'parameters' variable from main constructor. + """Validation check for 'parameters' variable from main constructor. Called within parent BasePattern constructor.""" valid_dict( - parameters, - str, - Any, - strict=False, - min_length=0, + parameters, + str, + Any, + strict=False, + min_length=0, hint="FileEventPattern.parameters" ) for k in parameters.keys(): valid_string( - k, + k, VALID_VARIABLE_NAME_CHARS, hint=f"FileEventPattern.parameters[{k}]" ) def _is_valid_output(self, outputs:Dict[str,str])->None: - """Validation check for 'output' variable from main constructor. + """Validation check for 'output' variable from main constructor. Called within parent BasePattern constructor.""" valid_dict( - outputs, - str, - str, - strict=False, + outputs, + str, + str, + strict=False, min_length=0, hint="FileEventPattern.outputs" ) for k in outputs.keys(): valid_string( - k, + k, VALID_VARIABLE_NAME_CHARS, hint=f"FileEventPattern.outputs[{k}]" ) @@ -167,9 +164,9 @@ class FileEventPattern(BasePattern): def _is_valid_event_mask(self, event_mask)->None: """Validation check for 'event_mask' variable from main constructor.""" valid_list( - event_mask, - str, - min_length=1, + event_mask, + str, + min_length=1, hint="FileEventPattern.event_mask" ) for mask in event_mask: @@ -193,18 +190,18 @@ class WatchdogMonitor(BaseMonitor): debug_level:int # Where print messages are sent _print_target:Any - def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern], - recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1, + def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern], + recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1, name:str="", print:Any=sys.stdout, logging:int=0)->None: - """WatchdogEventHandler Constructor. This uses the watchdog module to - monitor a directory and all its sub-directories. Watchdog will provide - the monitor with an caught events, with the monitor comparing them + """WatchdogEventHandler Constructor. This uses the watchdog module to + monitor a directory and all its sub-directories. Watchdog will provide + the monitor with an caught events, with the monitor comparing them against its rules, and informing the runner of match.""" super().__init__(patterns, recipes, name=name) self._is_valid_base_dir(base_dir) self.base_dir = base_dir check_type(settletime, int, hint="WatchdogMonitor.settletime") - self._print_target, self.debug_level = setup_debugging(print, logging) + self._print_target, self.debug_level = setup_debugging(print, logging) self.event_handler = WatchdogEventHandler(self, settletime=settletime) self.monitor = Observer() self.monitor.schedule( @@ -212,7 +209,7 @@ class WatchdogMonitor(BaseMonitor): self.base_dir, recursive=True ) - print_debug(self._print_target, self.debug_level, + print_debug(self._print_target, self.debug_level, "Created new WatchdogMonitor instance", DEBUG_INFO) if autostart: @@ -220,14 +217,14 @@ class WatchdogMonitor(BaseMonitor): def start(self)->None: """Function to start the monitor.""" - print_debug(self._print_target, self.debug_level, + print_debug(self._print_target, self.debug_level, "Starting WatchdogMonitor", DEBUG_INFO) self._apply_retroactive_rules() self.monitor.start() def stop(self)->None: """Function to stop the monitor.""" - print_debug(self._print_target, self.debug_level, + print_debug(self._print_target, self.debug_level, "Stopping WatchdogMonitor", DEBUG_INFO) self.monitor.stop() @@ -235,7 +232,7 @@ class WatchdogMonitor(BaseMonitor): """Function to determine if a given event matches the current rules.""" src_path = event.src_path - prepend = "dir_" if event.is_directory else "file_" + prepend = "dir_" if event.is_directory else "file_" event_types = [prepend+i for i in event.event_type] # Remove the base dir from the path as trigger paths are given relative @@ -248,12 +245,12 @@ class WatchdogMonitor(BaseMonitor): self._rules_lock.acquire() try: for rule in self._rules.values(): - + # Skip events not within the event mask if any(i in event_types for i in rule.pattern.event_mask) \ != True: continue - + # Use regex to match event paths against rule paths target_path = rule.pattern.triggering_path recursive_regexp = translate(target_path) @@ -273,10 +270,10 @@ class WatchdogMonitor(BaseMonitor): rule, self.base_dir, event.time_stamp, - get_hash(event.src_path, SHA256) + get_hash(event.src_path, SHA256) ) - print_debug(self._print_target, self.debug_level, - f"Event at {src_path} hit rule {rule.name}", + print_debug(self._print_target, self.debug_level, + f"Event at {src_path} hit rule {rule.name}", DEBUG_INFO) # Send the event to the runner self.send_event_to_runner(meow_event) @@ -288,17 +285,17 @@ class WatchdogMonitor(BaseMonitor): self._rules_lock.release() def _is_valid_base_dir(self, base_dir:str)->None: - """Validation check for 'base_dir' variable from main constructor. Is + """Validation check for 'base_dir' variable from main constructor. Is automatically called during initialisation.""" valid_dir_path(base_dir, must_exist=True) def _is_valid_patterns(self, patterns:Dict[str,FileEventPattern])->None: - """Validation check for 'patterns' variable from main constructor. Is + """Validation check for 'patterns' variable from main constructor. Is automatically called during initialisation.""" valid_dict(patterns, str, FileEventPattern, min_length=0, strict=False) def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None: - """Validation check for 'recipes' variable from main constructor. Is + """Validation check for 'recipes' variable from main constructor. Is automatically called during initialisation.""" valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False) @@ -309,7 +306,7 @@ class WatchdogMonitor(BaseMonitor): return [BaseRecipe] def _apply_retroactive_rule(self, rule:Rule)->None: - """Function to determine if a rule should be applied to the existing + """Function to determine if a rule should be applied to the existing file structure, were the file structure created/modified now.""" self._rules_lock.acquire() try: @@ -337,7 +334,7 @@ class WatchdogMonitor(BaseMonitor): time(), get_hash(globble, SHA256) ) - print_debug(self._print_target, self.debug_level, + print_debug(self._print_target, self.debug_level, f"Retroactive event for file at at {globble} hit rule " f"{rule.name}", DEBUG_INFO) # Send it to the runner @@ -349,7 +346,7 @@ class WatchdogMonitor(BaseMonitor): self._rules_lock.release() def _apply_retroactive_rules(self)->None: - """Function to determine if any rules should be applied to the existing + """Function to determine if any rules should be applied to the existing file structure, were the file structure created/modified now.""" for rule in self._rules.values(): self._apply_retroactive_rule(rule) @@ -366,8 +363,8 @@ class WatchdogEventHandler(PatternMatchingEventHandler): # A lock to solve race conditions on '_recent_jobs' _recent_jobs_lock:threading.Lock def __init__(self, monitor:WatchdogMonitor, settletime:int=1): - """WatchdogEventHandler Constructor. This inherits from watchdog - PatternMatchingEventHandler, and is used to catch events, then filter + """WatchdogEventHandler Constructor. This inherits from watchdog + PatternMatchingEventHandler, and is used to catch events, then filter out excessive events at the same location.""" super().__init__() self.monitor = monitor @@ -376,14 +373,14 @@ class WatchdogEventHandler(PatternMatchingEventHandler): self._recent_jobs_lock = threading.Lock() def threaded_handler(self, event): - """Function to determine if the given event shall be sent on to the - monitor. After each event we wait for '_settletime', to catch - subsequent events at the same location, so as to not swamp the system + """Function to determine if the given event shall be sent on to the + monitor. After each event we wait for '_settletime', to catch + subsequent events at the same location, so as to not swamp the system with repeated events.""" - + self._recent_jobs_lock.acquire() try: - if event.src_path in self._recent_jobs: + if event.src_path in self._recent_jobs: if event.time_stamp > self._recent_jobs[event.src_path][0]: self._recent_jobs[event.src_path][0] = event.time_stamp self._recent_jobs[event.src_path][1].add(event.event_type) @@ -395,7 +392,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler): [event.time_stamp, {event.event_type}] # If we have a closed event then short-cut the wait and send event - # immediately + # immediately if event.event_type == FILE_CLOSED_EVENT: self.monitor.match(event) self._recent_jobs_lock.release() @@ -423,9 +420,9 @@ class WatchdogEventHandler(PatternMatchingEventHandler): self.monitor.match(event) def handle_event(self, event): - """Handler function, called by all specific event functions. Will - attach a timestamp to the event immediately, and attempt to start a - threaded_handler so that the monitor can resume monitoring as soon as + """Handler function, called by all specific event functions. Will + attach a timestamp to the event immediately, and attempt to start a + threaded_handler so that the monitor can resume monitoring as soon as possible.""" event.time_stamp = time() @@ -440,7 +437,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler): waiting_for_threaded_resources = False except threading.ThreadError: sleep(1) - + def on_created(self, event): """Function called when a file created event occurs.""" self.handle_event(event) @@ -456,7 +453,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler): def on_deleted(self, event): """Function called when a file deleted event occurs.""" self.handle_event(event) - + def on_closed(self, event): """Function called when a file closed event occurs.""" self.handle_event(event)