NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Small spelling fix

Browse Source
This commit is contained in:
NikolajDanger
2023-05-09 10:14:39 +02:00
parent 83ee6b2d55
commit d45ab051dd
1 changed files with 68 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
patterns/file_event_pattern.py
View File

@ -1,6 +1,6 @@
""" """
This file contains definitions for a MEOW pattern based off of file events, This file contains definitions for a MEOW pattern based off of file events,
along with an appropriate monitor for said events. along with an appropriate monitor for said events.
Author(s): David Marchant Author(s): David Marchant
@ -49,16 +49,16 @@ WATCHDOG_HASH = "file_hash"
WATCHDOG_EVENT_KEYS = { WATCHDOG_EVENT_KEYS = {
WATCHDOG_BASE: str, WATCHDOG_BASE: str,
WATCHDOG_HASH: str, WATCHDOG_HASH: str,
**EVENT_KEYS **EVENT_KEYS
} }
def create_watchdog_event(path:str, rule:Any, base:str, time:float, def create_watchdog_event(path:str, rule:Any, base:str, time:float,
hash:str, extras:Dict[Any,Any]={})->Dict[Any,Any]: hash:str, extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW event dictionary.""" """Function to create a MEOW event dictionary."""
return create_event( return create_event(
EVENT_TYPE_WATCHDOG, EVENT_TYPE_WATCHDOG,
path, path,
rule, rule,
time, time,
extras={ extras={
@ -70,9 +70,6 @@ def create_watchdog_event(path:str, rule:Any, base:str, time:float,
} }
) )
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)
class FileEventPattern(BasePattern): class FileEventPattern(BasePattern):
# The path at which events will trigger this pattern # The path at which events will trigger this pattern
@ -81,11 +78,11 @@ class FileEventPattern(BasePattern):
triggering_file:str triggering_file:str
# Which types of event the pattern responds to # Which types of event the pattern responds to
event_mask:List[str] event_mask:List[str]
def __init__(self, name:str, triggering_path:str, recipe:str, def __init__(self, name:str, triggering_path:str, recipe:str,
triggering_file:str, event_mask:List[str]=_DEFAULT_MASK, triggering_file:str, event_mask:List[str]=_DEFAULT_MASK,
parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={}, parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={},
sweep:Dict[str,Any]={}): sweep:Dict[str,Any]={}):
"""FileEventPattern Constructor. This is used to match against file """FileEventPattern Constructor. This is used to match against file
system events, as caught by the python watchdog module.""" system events, as caught by the python watchdog module."""
super().__init__(name, recipe, parameters, outputs, sweep) super().__init__(name, recipe, parameters, outputs, sweep)
self._is_valid_triggering_path(triggering_path) self._is_valid_triggering_path(triggering_path)
@ -96,70 +93,70 @@ class FileEventPattern(BasePattern):
self.event_mask = event_mask self.event_mask = event_mask
def _is_valid_triggering_path(self, triggering_path:str)->None: def _is_valid_triggering_path(self, triggering_path:str)->None:
"""Validation check for 'triggering_path' variable from main """Validation check for 'triggering_path' variable from main
constructor.""" constructor."""
valid_string( valid_string(
triggering_path, triggering_path,
VALID_PATH_CHARS+'*', VALID_PATH_CHARS+'*',
min_length=1, min_length=1,
hint="FileEventPattern.triggering_path" hint="FileEventPattern.triggering_path"
) )
if len(triggering_path) < 1: if len(triggering_path) < 1:
raise ValueError ( raise ValueError (
f"triggiering path '{triggering_path}' is too short. " f"triggering path '{triggering_path}' is too short. "
"Minimum length is 1" "Minimum length is 1"
) )
def _is_valid_triggering_file(self, triggering_file:str)->None: def _is_valid_triggering_file(self, triggering_file:str)->None:
"""Validation check for 'triggering_file' variable from main """Validation check for 'triggering_file' variable from main
constructor.""" constructor."""
valid_string( valid_string(
triggering_file, triggering_file,
VALID_VARIABLE_NAME_CHARS, VALID_VARIABLE_NAME_CHARS,
hint="FileEventPattern.triggering_file" hint="FileEventPattern.triggering_file"
) )
def _is_valid_recipe(self, recipe:str)->None: def _is_valid_recipe(self, recipe:str)->None:
"""Validation check for 'recipe' variable from main constructor. """Validation check for 'recipe' variable from main constructor.
Called within parent BasePattern constructor.""" Called within parent BasePattern constructor."""
valid_string( valid_string(
recipe, recipe,
VALID_RECIPE_NAME_CHARS, VALID_RECIPE_NAME_CHARS,
hint="FileEventPattern.recipe" hint="FileEventPattern.recipe"
) )
def _is_valid_parameters(self, parameters:Dict[str,Any])->None: def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
"""Validation check for 'parameters' variable from main constructor. """Validation check for 'parameters' variable from main constructor.
Called within parent BasePattern constructor.""" Called within parent BasePattern constructor."""
valid_dict( valid_dict(
parameters, parameters,
str, str,
Any, Any,
strict=False, strict=False,
min_length=0, min_length=0,
hint="FileEventPattern.parameters" hint="FileEventPattern.parameters"
) )
for k in parameters.keys(): for k in parameters.keys():
valid_string( valid_string(
k, k,
VALID_VARIABLE_NAME_CHARS, VALID_VARIABLE_NAME_CHARS,
hint=f"FileEventPattern.parameters[{k}]" hint=f"FileEventPattern.parameters[{k}]"
) )
def _is_valid_output(self, outputs:Dict[str,str])->None: def _is_valid_output(self, outputs:Dict[str,str])->None:
"""Validation check for 'output' variable from main constructor. """Validation check for 'output' variable from main constructor.
Called within parent BasePattern constructor.""" Called within parent BasePattern constructor."""
valid_dict( valid_dict(
outputs, outputs,
str, str,
str, str,
strict=False, strict=False,
min_length=0, min_length=0,
hint="FileEventPattern.outputs" hint="FileEventPattern.outputs"
) )
for k in outputs.keys(): for k in outputs.keys():
valid_string( valid_string(
k, k,
VALID_VARIABLE_NAME_CHARS, VALID_VARIABLE_NAME_CHARS,
hint=f"FileEventPattern.outputs[{k}]" hint=f"FileEventPattern.outputs[{k}]"
) )
@ -167,9 +164,9 @@ class FileEventPattern(BasePattern):
def _is_valid_event_mask(self, event_mask)->None: def _is_valid_event_mask(self, event_mask)->None:
"""Validation check for 'event_mask' variable from main constructor.""" """Validation check for 'event_mask' variable from main constructor."""
valid_list( valid_list(
event_mask, event_mask,
str, str,
min_length=1, min_length=1,
hint="FileEventPattern.event_mask" hint="FileEventPattern.event_mask"
) )
for mask in event_mask: for mask in event_mask:
@ -193,18 +190,18 @@ class WatchdogMonitor(BaseMonitor):
debug_level:int debug_level:int
# Where print messages are sent # Where print messages are sent
_print_target:Any _print_target:Any
def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern], def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern],
recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1, recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1,
name:str="", print:Any=sys.stdout, logging:int=0)->None: name:str="", print:Any=sys.stdout, logging:int=0)->None:
"""WatchdogEventHandler Constructor. This uses the watchdog module to """WatchdogEventHandler Constructor. This uses the watchdog module to
monitor a directory and all its sub-directories. Watchdog will provide monitor a directory and all its sub-directories. Watchdog will provide
the monitor with an caught events, with the monitor comparing them the monitor with an caught events, with the monitor comparing them
against its rules, and informing the runner of match.""" against its rules, and informing the runner of match."""
super().__init__(patterns, recipes, name=name) super().__init__(patterns, recipes, name=name)
self._is_valid_base_dir(base_dir) self._is_valid_base_dir(base_dir)
self.base_dir = base_dir self.base_dir = base_dir
check_type(settletime, int, hint="WatchdogMonitor.settletime") check_type(settletime, int, hint="WatchdogMonitor.settletime")
self._print_target, self.debug_level = setup_debugging(print, logging) self._print_target, self.debug_level = setup_debugging(print, logging)
self.event_handler = WatchdogEventHandler(self, settletime=settletime) self.event_handler = WatchdogEventHandler(self, settletime=settletime)
self.monitor = Observer() self.monitor = Observer()
self.monitor.schedule( self.monitor.schedule(
@ -212,7 +209,7 @@ class WatchdogMonitor(BaseMonitor):
self.base_dir, self.base_dir,
recursive=True recursive=True
) )
print_debug(self._print_target, self.debug_level, print_debug(self._print_target, self.debug_level,
"Created new WatchdogMonitor instance", DEBUG_INFO) "Created new WatchdogMonitor instance", DEBUG_INFO)
if autostart: if autostart:
@ -220,14 +217,14 @@ class WatchdogMonitor(BaseMonitor):
def start(self)->None: def start(self)->None:
"""Function to start the monitor.""" """Function to start the monitor."""
print_debug(self._print_target, self.debug_level, print_debug(self._print_target, self.debug_level,
"Starting WatchdogMonitor", DEBUG_INFO) "Starting WatchdogMonitor", DEBUG_INFO)
self._apply_retroactive_rules() self._apply_retroactive_rules()
self.monitor.start() self.monitor.start()
def stop(self)->None: def stop(self)->None:
"""Function to stop the monitor.""" """Function to stop the monitor."""
print_debug(self._print_target, self.debug_level, print_debug(self._print_target, self.debug_level,
"Stopping WatchdogMonitor", DEBUG_INFO) "Stopping WatchdogMonitor", DEBUG_INFO)
self.monitor.stop() self.monitor.stop()
@ -235,7 +232,7 @@ class WatchdogMonitor(BaseMonitor):
"""Function to determine if a given event matches the current rules.""" """Function to determine if a given event matches the current rules."""
src_path = event.src_path src_path = event.src_path
prepend = "dir_" if event.is_directory else "file_" prepend = "dir_" if event.is_directory else "file_"
event_types = [prepend+i for i in event.event_type] event_types = [prepend+i for i in event.event_type]
# Remove the base dir from the path as trigger paths are given relative # Remove the base dir from the path as trigger paths are given relative
@ -248,12 +245,12 @@ class WatchdogMonitor(BaseMonitor):
self._rules_lock.acquire() self._rules_lock.acquire()
try: try:
for rule in self._rules.values(): for rule in self._rules.values():
# Skip events not within the event mask # Skip events not within the event mask
if any(i in event_types for i in rule.pattern.event_mask) \ if any(i in event_types for i in rule.pattern.event_mask) \
!= True: != True:
continue continue
# Use regex to match event paths against rule paths # Use regex to match event paths against rule paths
target_path = rule.pattern.triggering_path target_path = rule.pattern.triggering_path
recursive_regexp = translate(target_path) recursive_regexp = translate(target_path)
@ -273,10 +270,10 @@ class WatchdogMonitor(BaseMonitor):
rule, rule,
self.base_dir, self.base_dir,
event.time_stamp, event.time_stamp,
get_hash(event.src_path, SHA256) get_hash(event.src_path, SHA256)
) )
print_debug(self._print_target, self.debug_level, print_debug(self._print_target, self.debug_level,
f"Event at {src_path} hit rule {rule.name}", f"Event at {src_path} hit rule {rule.name}",
DEBUG_INFO) DEBUG_INFO)
# Send the event to the runner # Send the event to the runner
self.send_event_to_runner(meow_event) self.send_event_to_runner(meow_event)
@ -288,17 +285,17 @@ class WatchdogMonitor(BaseMonitor):
self._rules_lock.release() self._rules_lock.release()
def _is_valid_base_dir(self, base_dir:str)->None: def _is_valid_base_dir(self, base_dir:str)->None:
"""Validation check for 'base_dir' variable from main constructor. Is """Validation check for 'base_dir' variable from main constructor. Is
automatically called during initialisation.""" automatically called during initialisation."""
valid_dir_path(base_dir, must_exist=True) valid_dir_path(base_dir, must_exist=True)
def _is_valid_patterns(self, patterns:Dict[str,FileEventPattern])->None: def _is_valid_patterns(self, patterns:Dict[str,FileEventPattern])->None:
"""Validation check for 'patterns' variable from main constructor. Is """Validation check for 'patterns' variable from main constructor. Is
automatically called during initialisation.""" automatically called during initialisation."""
valid_dict(patterns, str, FileEventPattern, min_length=0, strict=False) valid_dict(patterns, str, FileEventPattern, min_length=0, strict=False)
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None: def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
"""Validation check for 'recipes' variable from main constructor. Is """Validation check for 'recipes' variable from main constructor. Is
automatically called during initialisation.""" automatically called during initialisation."""
valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False) valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False)
@ -309,7 +306,7 @@ class WatchdogMonitor(BaseMonitor):
return [BaseRecipe] return [BaseRecipe]
def _apply_retroactive_rule(self, rule:Rule)->None: def _apply_retroactive_rule(self, rule:Rule)->None:
"""Function to determine if a rule should be applied to the existing """Function to determine if a rule should be applied to the existing
file structure, were the file structure created/modified now.""" file structure, were the file structure created/modified now."""
self._rules_lock.acquire() self._rules_lock.acquire()
try: try:
@ -337,7 +334,7 @@ class WatchdogMonitor(BaseMonitor):
time(), time(),
get_hash(globble, SHA256) get_hash(globble, SHA256)
) )
print_debug(self._print_target, self.debug_level, print_debug(self._print_target, self.debug_level,
f"Retroactive event for file at at {globble} hit rule " f"Retroactive event for file at at {globble} hit rule "
f"{rule.name}", DEBUG_INFO) f"{rule.name}", DEBUG_INFO)
# Send it to the runner # Send it to the runner
@ -349,7 +346,7 @@ class WatchdogMonitor(BaseMonitor):
self._rules_lock.release() self._rules_lock.release()
def _apply_retroactive_rules(self)->None: def _apply_retroactive_rules(self)->None:
"""Function to determine if any rules should be applied to the existing """Function to determine if any rules should be applied to the existing
file structure, were the file structure created/modified now.""" file structure, were the file structure created/modified now."""
for rule in self._rules.values(): for rule in self._rules.values():
self._apply_retroactive_rule(rule) self._apply_retroactive_rule(rule)
@ -366,8 +363,8 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
# A lock to solve race conditions on '_recent_jobs' # A lock to solve race conditions on '_recent_jobs'
_recent_jobs_lock:threading.Lock _recent_jobs_lock:threading.Lock
def __init__(self, monitor:WatchdogMonitor, settletime:int=1): def __init__(self, monitor:WatchdogMonitor, settletime:int=1):
"""WatchdogEventHandler Constructor. This inherits from watchdog """WatchdogEventHandler Constructor. This inherits from watchdog
PatternMatchingEventHandler, and is used to catch events, then filter PatternMatchingEventHandler, and is used to catch events, then filter
out excessive events at the same location.""" out excessive events at the same location."""
super().__init__() super().__init__()
self.monitor = monitor self.monitor = monitor
@ -376,14 +373,14 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
self._recent_jobs_lock = threading.Lock() self._recent_jobs_lock = threading.Lock()
def threaded_handler(self, event): def threaded_handler(self, event):
"""Function to determine if the given event shall be sent on to the """Function to determine if the given event shall be sent on to the
monitor. After each event we wait for '_settletime', to catch monitor. After each event we wait for '_settletime', to catch
subsequent events at the same location, so as to not swamp the system subsequent events at the same location, so as to not swamp the system
with repeated events.""" with repeated events."""
self._recent_jobs_lock.acquire() self._recent_jobs_lock.acquire()
try: try:
if event.src_path in self._recent_jobs: if event.src_path in self._recent_jobs:
if event.time_stamp > self._recent_jobs[event.src_path][0]: if event.time_stamp > self._recent_jobs[event.src_path][0]:
self._recent_jobs[event.src_path][0] = event.time_stamp self._recent_jobs[event.src_path][0] = event.time_stamp
self._recent_jobs[event.src_path][1].add(event.event_type) self._recent_jobs[event.src_path][1].add(event.event_type)
@ -395,7 +392,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
[event.time_stamp, {event.event_type}] [event.time_stamp, {event.event_type}]
# If we have a closed event then short-cut the wait and send event # If we have a closed event then short-cut the wait and send event
# immediately # immediately
if event.event_type == FILE_CLOSED_EVENT: if event.event_type == FILE_CLOSED_EVENT:
self.monitor.match(event) self.monitor.match(event)
self._recent_jobs_lock.release() self._recent_jobs_lock.release()
@ -423,9 +420,9 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
self.monitor.match(event) self.monitor.match(event)
def handle_event(self, event): def handle_event(self, event):
"""Handler function, called by all specific event functions. Will """Handler function, called by all specific event functions. Will
attach a timestamp to the event immediately, and attempt to start a attach a timestamp to the event immediately, and attempt to start a
threaded_handler so that the monitor can resume monitoring as soon as threaded_handler so that the monitor can resume monitoring as soon as
possible.""" possible."""
event.time_stamp = time() event.time_stamp = time()
@ -440,7 +437,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
waiting_for_threaded_resources = False waiting_for_threaded_resources = False
except threading.ThreadError: except threading.ThreadError:
sleep(1) sleep(1)
def on_created(self, event): def on_created(self, event):
"""Function called when a file created event occurs.""" """Function called when a file created event occurs."""
self.handle_event(event) self.handle_event(event)
@ -456,7 +453,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
def on_deleted(self, event): def on_deleted(self, event):
"""Function called when a file deleted event occurs.""" """Function called when a file deleted event occurs."""
self.handle_event(event) self.handle_event(event)
def on_closed(self, event): def on_closed(self, event):
"""Function called when a file closed event occurs.""" """Function called when a file closed event occurs."""
self.handle_event(event) self.handle_event(event)