NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Small spelling fix

Browse Source
This commit is contained in:
NikolajDanger
2023-05-09 10:14:39 +02:00
parent 83ee6b2d55
commit d45ab051dd
1 changed files with 68 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
patterns/file_event_pattern.py
View File

@ -1,6 +1,6 @@
"""
This file contains definitions for a MEOW pattern based off of file events,
This file contains definitions for a MEOW pattern based off of file events,
along with an appropriate monitor for said events.
Author(s): David Marchant
@ -49,16 +49,16 @@ WATCHDOG_HASH = "file_hash"
WATCHDOG_EVENT_KEYS = {
WATCHDOG_BASE: str,
WATCHDOG_HASH: str,
WATCHDOG_HASH: str,
**EVENT_KEYS
}
def create_watchdog_event(path:str, rule:Any, base:str, time:float,
def create_watchdog_event(path:str, rule:Any, base:str, time:float,
hash:str, extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW event dictionary."""
return create_event(
EVENT_TYPE_WATCHDOG,
path,
EVENT_TYPE_WATCHDOG,
path,
rule,
time,
extras={
@ -70,9 +70,6 @@ def create_watchdog_event(path:str, rule:Any, base:str, time:float,
}
)
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)
class FileEventPattern(BasePattern):
# The path at which events will trigger this pattern
@ -81,11 +78,11 @@ class FileEventPattern(BasePattern):
triggering_file:str
# Which types of event the pattern responds to
event_mask:List[str]
def __init__(self, name:str, triggering_path:str, recipe:str,
triggering_file:str, event_mask:List[str]=_DEFAULT_MASK,
parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={},
def __init__(self, name:str, triggering_path:str, recipe:str,
triggering_file:str, event_mask:List[str]=_DEFAULT_MASK,
parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={},
sweep:Dict[str,Any]={}):
"""FileEventPattern Constructor. This is used to match against file
"""FileEventPattern Constructor. This is used to match against file
system events, as caught by the python watchdog module."""
super().__init__(name, recipe, parameters, outputs, sweep)
self._is_valid_triggering_path(triggering_path)
@ -96,70 +93,70 @@ class FileEventPattern(BasePattern):
self.event_mask = event_mask
def _is_valid_triggering_path(self, triggering_path:str)->None:
"""Validation check for 'triggering_path' variable from main
"""Validation check for 'triggering_path' variable from main
constructor."""
valid_string(
triggering_path,
VALID_PATH_CHARS+'*',
min_length=1,
triggering_path,
VALID_PATH_CHARS+'*',
min_length=1,
hint="FileEventPattern.triggering_path"
)
if len(triggering_path) < 1:
raise ValueError (
f"triggiering path '{triggering_path}' is too short. "
f"triggering path '{triggering_path}' is too short. "
"Minimum length is 1"
)
def _is_valid_triggering_file(self, triggering_file:str)->None:
"""Validation check for 'triggering_file' variable from main
"""Validation check for 'triggering_file' variable from main
constructor."""
valid_string(
triggering_file,
triggering_file,
VALID_VARIABLE_NAME_CHARS,
hint="FileEventPattern.triggering_file"
)
def _is_valid_recipe(self, recipe:str)->None:
"""Validation check for 'recipe' variable from main constructor.
"""Validation check for 'recipe' variable from main constructor.
Called within parent BasePattern constructor."""
valid_string(
recipe,
recipe,
VALID_RECIPE_NAME_CHARS,
hint="FileEventPattern.recipe"
)
def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
"""Validation check for 'parameters' variable from main constructor.
"""Validation check for 'parameters' variable from main constructor.
Called within parent BasePattern constructor."""
valid_dict(
parameters,
str,
Any,
strict=False,
min_length=0,
parameters,
str,
Any,
strict=False,
min_length=0,
hint="FileEventPattern.parameters"
)
for k in parameters.keys():
valid_string(
k,
k,
VALID_VARIABLE_NAME_CHARS,
hint=f"FileEventPattern.parameters[{k}]"
)
def _is_valid_output(self, outputs:Dict[str,str])->None:
"""Validation check for 'output' variable from main constructor.
"""Validation check for 'output' variable from main constructor.
Called within parent BasePattern constructor."""
valid_dict(
outputs,
str,
str,
strict=False,
outputs,
str,
str,
strict=False,
min_length=0,
hint="FileEventPattern.outputs"
)
for k in outputs.keys():
valid_string(
k,
k,
VALID_VARIABLE_NAME_CHARS,
hint=f"FileEventPattern.outputs[{k}]"
)
@ -167,9 +164,9 @@ class FileEventPattern(BasePattern):
def _is_valid_event_mask(self, event_mask)->None:
"""Validation check for 'event_mask' variable from main constructor."""
valid_list(
event_mask,
str,
min_length=1,
event_mask,
str,
min_length=1,
hint="FileEventPattern.event_mask"
)
for mask in event_mask:
@ -193,18 +190,18 @@ class WatchdogMonitor(BaseMonitor):
debug_level:int
# Where print messages are sent
_print_target:Any
def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern],
recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1,
def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern],
recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1,
name:str="", print:Any=sys.stdout, logging:int=0)->None:
"""WatchdogEventHandler Constructor. This uses the watchdog module to
monitor a directory and all its sub-directories. Watchdog will provide
the monitor with an caught events, with the monitor comparing them
"""WatchdogEventHandler Constructor. This uses the watchdog module to
monitor a directory and all its sub-directories. Watchdog will provide
the monitor with an caught events, with the monitor comparing them
against its rules, and informing the runner of match."""
super().__init__(patterns, recipes, name=name)
self._is_valid_base_dir(base_dir)
self.base_dir = base_dir
check_type(settletime, int, hint="WatchdogMonitor.settletime")
self._print_target, self.debug_level = setup_debugging(print, logging)
self._print_target, self.debug_level = setup_debugging(print, logging)
self.event_handler = WatchdogEventHandler(self, settletime=settletime)
self.monitor = Observer()
self.monitor.schedule(
@ -212,7 +209,7 @@ class WatchdogMonitor(BaseMonitor):
self.base_dir,
recursive=True
)
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
"Created new WatchdogMonitor instance", DEBUG_INFO)
if autostart:
@ -220,14 +217,14 @@ class WatchdogMonitor(BaseMonitor):
def start(self)->None:
"""Function to start the monitor."""
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
"Starting WatchdogMonitor", DEBUG_INFO)
self._apply_retroactive_rules()
self.monitor.start()
def stop(self)->None:
"""Function to stop the monitor."""
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
"Stopping WatchdogMonitor", DEBUG_INFO)
self.monitor.stop()
@ -235,7 +232,7 @@ class WatchdogMonitor(BaseMonitor):
"""Function to determine if a given event matches the current rules."""
src_path = event.src_path
prepend = "dir_" if event.is_directory else "file_"
prepend = "dir_" if event.is_directory else "file_"
event_types = [prepend+i for i in event.event_type]
# Remove the base dir from the path as trigger paths are given relative
@ -248,12 +245,12 @@ class WatchdogMonitor(BaseMonitor):
self._rules_lock.acquire()
try:
for rule in self._rules.values():
# Skip events not within the event mask
if any(i in event_types for i in rule.pattern.event_mask) \
!= True:
continue
# Use regex to match event paths against rule paths
target_path = rule.pattern.triggering_path
recursive_regexp = translate(target_path)
@ -273,10 +270,10 @@ class WatchdogMonitor(BaseMonitor):
rule,
self.base_dir,
event.time_stamp,
get_hash(event.src_path, SHA256)
get_hash(event.src_path, SHA256)
)
print_debug(self._print_target, self.debug_level,
f"Event at {src_path} hit rule {rule.name}",
print_debug(self._print_target, self.debug_level,
f"Event at {src_path} hit rule {rule.name}",
DEBUG_INFO)
# Send the event to the runner
self.send_event_to_runner(meow_event)
@ -288,17 +285,17 @@ class WatchdogMonitor(BaseMonitor):
self._rules_lock.release()
def _is_valid_base_dir(self, base_dir:str)->None:
"""Validation check for 'base_dir' variable from main constructor. Is
"""Validation check for 'base_dir' variable from main constructor. Is
automatically called during initialisation."""
valid_dir_path(base_dir, must_exist=True)
def _is_valid_patterns(self, patterns:Dict[str,FileEventPattern])->None:
"""Validation check for 'patterns' variable from main constructor. Is
"""Validation check for 'patterns' variable from main constructor. Is
automatically called during initialisation."""
valid_dict(patterns, str, FileEventPattern, min_length=0, strict=False)
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
"""Validation check for 'recipes' variable from main constructor. Is
"""Validation check for 'recipes' variable from main constructor. Is
automatically called during initialisation."""
valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False)
@ -309,7 +306,7 @@ class WatchdogMonitor(BaseMonitor):
return [BaseRecipe]
def _apply_retroactive_rule(self, rule:Rule)->None:
"""Function to determine if a rule should be applied to the existing
"""Function to determine if a rule should be applied to the existing
file structure, were the file structure created/modified now."""
self._rules_lock.acquire()
try:
@ -337,7 +334,7 @@ class WatchdogMonitor(BaseMonitor):
time(),
get_hash(globble, SHA256)
)
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
f"Retroactive event for file at at {globble} hit rule "
f"{rule.name}", DEBUG_INFO)
# Send it to the runner
@ -349,7 +346,7 @@ class WatchdogMonitor(BaseMonitor):
self._rules_lock.release()
def _apply_retroactive_rules(self)->None:
"""Function to determine if any rules should be applied to the existing
"""Function to determine if any rules should be applied to the existing
file structure, were the file structure created/modified now."""
for rule in self._rules.values():
self._apply_retroactive_rule(rule)
@ -366,8 +363,8 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
# A lock to solve race conditions on '_recent_jobs'
_recent_jobs_lock:threading.Lock
def __init__(self, monitor:WatchdogMonitor, settletime:int=1):
"""WatchdogEventHandler Constructor. This inherits from watchdog
PatternMatchingEventHandler, and is used to catch events, then filter
"""WatchdogEventHandler Constructor. This inherits from watchdog
PatternMatchingEventHandler, and is used to catch events, then filter
out excessive events at the same location."""
super().__init__()
self.monitor = monitor
@ -376,14 +373,14 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
self._recent_jobs_lock = threading.Lock()
def threaded_handler(self, event):
"""Function to determine if the given event shall be sent on to the
monitor. After each event we wait for '_settletime', to catch
subsequent events at the same location, so as to not swamp the system
"""Function to determine if the given event shall be sent on to the
monitor. After each event we wait for '_settletime', to catch
subsequent events at the same location, so as to not swamp the system
with repeated events."""
self._recent_jobs_lock.acquire()
try:
if event.src_path in self._recent_jobs:
if event.src_path in self._recent_jobs:
if event.time_stamp > self._recent_jobs[event.src_path][0]:
self._recent_jobs[event.src_path][0] = event.time_stamp
self._recent_jobs[event.src_path][1].add(event.event_type)
@ -395,7 +392,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
[event.time_stamp, {event.event_type}]
# If we have a closed event then short-cut the wait and send event
# immediately
# immediately
if event.event_type == FILE_CLOSED_EVENT:
self.monitor.match(event)
self._recent_jobs_lock.release()
@ -423,9 +420,9 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
self.monitor.match(event)
def handle_event(self, event):
"""Handler function, called by all specific event functions. Will
attach a timestamp to the event immediately, and attempt to start a
threaded_handler so that the monitor can resume monitoring as soon as
"""Handler function, called by all specific event functions. Will
attach a timestamp to the event immediately, and attempt to start a
threaded_handler so that the monitor can resume monitoring as soon as
possible."""
event.time_stamp = time()
@ -440,7 +437,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
waiting_for_threaded_resources = False
except threading.ThreadError:
sleep(1)
def on_created(self, event):
"""Function called when a file created event occurs."""
self.handle_event(event)
@ -456,7 +453,7 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
def on_deleted(self, event):
"""Function called when a file deleted event occurs."""
self.handle_event(event)
def on_closed(self, event):
"""Function called when a file closed event occurs."""
self.handle_event(event)