NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactored monitor handler interaction to better allow differing event types in same system

Browse Source
This commit is contained in:
PatchOfScotland
2023-01-13 18:04:50 +01:00
parent e9519d718f
commit d9004394c1
11 changed files with 357 additions and 355 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
core/correctness/validation.py
View File

@ -3,7 +3,7 @@ from inspect import signature
from os.path import sep, exists, isfile, isdir, dirname
from typing import Any, _SpecialForm, Union, Tuple, get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, EVENT_TYPE
def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
or_none:bool=False)->None:
@ -178,3 +178,8 @@ def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:
"'write' function")
return print, logging
def valid_event(event)->None:
check_type(event, dict)
if not EVENT_TYPE in event.keys():
raise KeyError(f"Events require key '{EVENT_TYPE}'")

9
core/correctness/vars.py
View File

@ -187,7 +187,14 @@ APPENDING_NOTEBOOK = {
"nbformat_minor": 4
}
# events
# meow events
EVENT_TYPE = "meow_event_type"
WATCHDOG_TYPE = "watchdog"
WATCHDOG_SRC = "src_path"
WATCHDOG_BASE = "monitor_base"
WATCHDOG_RULE = "rule_name"
# inotify events
FILE_CREATE_EVENT = "file_created"
FILE_MODIFY_EVENT = "file_modified"
FILE_MOVED_EVENT = "file_moved"

67
core/functionality.py
View File

@ -14,28 +14,12 @@ from papermill.translators import papermill_translators
from typing import Any, Union
from random import SystemRandom
from core.meow import BasePattern, BaseRecipe, BaseRule
#from core.meow import BasePattern, BaseRecipe, BaseRule
from core.correctness.validation import check_type, valid_dict, valid_list, \
valid_existing_file_path, valid_path
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
VALID_CHANNELS, HASH_BUFFER_SIZE, SHA256, DEBUG_WARNING, DEBUG_ERROR, \
DEBUG_INFO
def check_pattern_dict(patterns, min_length=1):
valid_dict(patterns, str, BasePattern, strict=False, min_length=min_length)
for k, v in patterns.items():
if k != v.name:
raise KeyError(f"Key '{k}' indexes unexpected Pattern '{v.name}' "
"Pattern dictionaries must be keyed with the name of the "
"Pattern.")
def check_recipe_dict(recipes, min_length=1):
valid_dict(recipes, str, BaseRecipe, strict=False, min_length=min_length)
for k, v in recipes.items():
if k != v.name:
raise KeyError(f"Key '{k}' indexes unexpected Recipe '{v.name}' "
"Recipe dictionaries must be keyed with the name of the "
"Recipe.")
DEBUG_INFO, EVENT_TYPE
def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
charset:str=CHAR_UPPERCASE+CHAR_LOWERCASE, attempts:int=24):
@ -48,45 +32,6 @@ def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
raise ValueError(f"Could not generate ID unique from '{existing_ids}' "
f"using values '{charset}' and length of '{length}'.")
def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
recipes:Union[dict[str,BaseRecipe],list[BaseRecipe]],
new_rules:list[BaseRule]=[])->dict[str,BaseRule]:
check_type(patterns, dict, alt_types=[list])
check_type(recipes, dict, alt_types=[list])
valid_list(new_rules, BaseRule, min_length=0)
if isinstance(patterns, list):
valid_list(patterns, BasePattern, min_length=0)
patterns = {pattern.name:pattern for pattern in patterns}
else:
check_pattern_dict(patterns, min_length=0)
if isinstance(recipes, list):
valid_list(recipes, BaseRecipe, min_length=0)
recipes = {recipe.name:recipe for recipe in recipes}
else:
check_recipe_dict(recipes, min_length=0)
# Imported here to avoid circular imports at top of file
import rules
rules = {}
all_rules ={(r.pattern_type, r.recipe_type):r for r in [r[1] \
for r in inspect.getmembers(sys.modules["rules"], inspect.isclass) \
if (issubclass(r[1], BaseRule))]}
for pattern in patterns.values():
if pattern.recipe in recipes:
key = (type(pattern).__name__,
type(recipes[pattern.recipe]).__name__)
if (key) in all_rules:
rule = all_rules[key](
generate_id(prefix="Rule_"),
pattern,
recipes[pattern.recipe]
)
rules[rule.name] = rule
return rules
def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]:
all_connections = [i for i in inputs if type(i) is Connection] \
+ [i._reader for i in inputs if type(i) is Queue]
@ -133,6 +78,8 @@ def rmtree(directory:str):
:return: No return
"""
if not os.path.exists(directory):
return
for root, dirs, files in os.walk(directory, topdown=False):
for file in files:
os.remove(os.path.join(root, file))
@ -295,4 +242,8 @@ def print_debug(print_target, debug_level, msg, level)->None:
status = "INFO"
elif level == DEBUG_WARNING:
status = "WARNING"
print(f"{status}: {msg}", file=print_target)
print(f"{status}: {msg}", file=print_target)
def create_event(event_type:str, source:dict[Any,Any]={})->dict[Any,Any]:
return {**source, EVENT_TYPE: event_type}

152
core/meow.py
View File

@ -1,11 +1,17 @@
from typing import Any
import inspect
import sys
import threading
from multiprocessing import Pipe
from typing import Any, Union
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
get_drt_imp_msg
get_drt_imp_msg, DEBUG_WARNING, DEBUG_INFO, DEBUG_ERROR
from core.correctness.validation import valid_string, check_type, \
check_implementation
check_implementation, valid_list, valid_dict, setup_debugging
from core.functionality import print_debug, wait, generate_id
class BaseRecipe:
@ -127,15 +133,11 @@ class BaseRule:
class BaseMonitor:
rules: dict[str, BaseRule]
report: VALID_CHANNELS
def __init__(self, rules:dict[str,BaseRule],
report:VALID_CHANNELS)->None:
to_runner: VALID_CHANNELS
def __init__(self, rules:dict[str,BaseRule])->None:
check_implementation(type(self).start, BaseMonitor)
check_implementation(type(self).stop, BaseMonitor)
check_implementation(type(self)._is_valid_report, BaseMonitor)
check_implementation(type(self)._is_valid_rules, BaseMonitor)
self._is_valid_report(report)
self.report = report
self._is_valid_rules(rules)
self.rules = rules
@ -145,9 +147,6 @@ class BaseMonitor:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_report(self, report:VALID_CHANNELS)->None:
pass
def _is_valid_rules(self, rules:dict[str,BaseRule])->None:
pass
@ -159,14 +158,9 @@ class BaseMonitor:
class BaseHandler:
inputs:Any
def __init__(self, inputs:list[VALID_CHANNELS]) -> None:
check_implementation(type(self).start, BaseHandler)
check_implementation(type(self).stop, BaseHandler)
def __init__(self) -> None:
check_implementation(type(self).handle, BaseHandler)
check_implementation(type(self)._is_valid_inputs, BaseHandler)
self._is_valid_inputs(inputs)
self.inputs = inputs
check_implementation(type(self).valid_event_types, BaseHandler)
def __new__(cls, *args, **kwargs):
if cls is BaseHandler:
@ -174,40 +168,134 @@ class BaseHandler:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_inputs(self, inputs:Any)->None:
def valid_event_types(self)->list[str]:
pass
def handle(self, event:Any, rule:BaseRule)->None:
pass
def start(self)->None:
pass
def stop(self)->None:
def handle(self, event:Any)->None:
pass
# TODO reformat to allow for updated monitor / handler interaction
# TODO expand this to allow for lists of monitors / handlers
class MeowRunner:
monitor:BaseMonitor
handler:BaseHandler
def __init__(self, monitor:BaseMonitor, handler:BaseHandler) -> None:
from_monitor: VALID_CHANNELS
def __init__(self, monitor:BaseMonitor, handler:BaseHandler,
print:Any=sys.stdout, logging:int=0) -> None:
self._is_valid_monitor(monitor)
self.monitor = monitor
monitor_to_runner_reader, monitor_to_runner_writer = Pipe()
self.monitor.to_runner = monitor_to_runner_writer
self.from_monitor = monitor_to_runner_reader
self._is_valid_handler(handler)
self.handler = handler
self._stop_pipe = Pipe()
self._worker = None
self._print_target, self.debug_level = setup_debugging(print, logging)
def run(self)->None:
all_inputs = [self.from_monitor, self._stop_pipe[0]]
while True:
ready = wait(all_inputs)
if self._stop_pipe[0] in ready:
return
else:
message = self.from_monitor.recv()
event = message
self.handler.handle(event)
def start(self)->None:
self.monitor.start()
if hasattr(self.handler, "start"):
self.handler.start()
#if hasattr(self.handler, "start"):
# self.handler.start()
if self._worker is None:
self._worker = threading.Thread(
target=self.run,
args=[])
self._worker.daemon = True
self._worker.start()
print_debug(self._print_target, self.debug_level,
"Starting MeowRunner run...", DEBUG_INFO)
else:
msg = "Repeated calls to start have no effect."
print_debug(self._print_target, self.debug_level,
msg, DEBUG_WARNING)
raise RuntimeWarning(msg)
def stop(self)->None:
self.monitor.stop()
if hasattr(self.handler, "stop"):
self.handler.stop()
#if hasattr(self.handler, "stop"):
# self.handler.stop()
if self._worker is None:
msg = "Cannot stop thread that is not started."
print_debug(self._print_target, self.debug_level,
msg, DEBUG_WARNING)
raise RuntimeWarning(msg)
else:
self._stop_pipe[1].send(1)
self._worker.join()
print_debug(self._print_target, self.debug_level,
"Worker thread stopped", DEBUG_INFO)
def _is_valid_monitor(self, monitor:BaseMonitor)->None:
check_type(monitor, BaseMonitor)
def _is_valid_handler(self, handler:BaseHandler)->None:
check_type(handler, BaseHandler)
def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
recipes:Union[dict[str,BaseRecipe],list[BaseRecipe]],
new_rules:list[BaseRule]=[])->dict[str,BaseRule]:
check_type(patterns, dict, alt_types=[list])
check_type(recipes, dict, alt_types=[list])
valid_list(new_rules, BaseRule, min_length=0)
if isinstance(patterns, list):
valid_list(patterns, BasePattern, min_length=0)
patterns = {pattern.name:pattern for pattern in patterns}
else:
valid_dict(patterns, str, BasePattern, strict=False, min_length=0)
for k, v in patterns.items():
if k != v.name:
raise KeyError(
f"Key '{k}' indexes unexpected Pattern '{v.name}' "
"Pattern dictionaries must be keyed with the name of the "
"Pattern.")
if isinstance(recipes, list):
valid_list(recipes, BaseRecipe, min_length=0)
recipes = {recipe.name:recipe for recipe in recipes}
else:
valid_dict(recipes, str, BaseRecipe, strict=False, min_length=0)
for k, v in recipes.items():
if k != v.name:
raise KeyError(
f"Key '{k}' indexes unexpected Recipe '{v.name}' "
"Recipe dictionaries must be keyed with the name of the "
"Recipe.")
# Imported here to avoid circular imports at top of file
import rules
rules = {}
all_rules ={(r.pattern_type, r.recipe_type):r for r in [r[1] \
for r in inspect.getmembers(sys.modules["rules"], inspect.isclass) \
if (issubclass(r[1], BaseRule))]}
for pattern in patterns.values():
if pattern.recipe in recipes:
key = (type(pattern).__name__,
type(recipes[pattern.recipe]).__name__)
if (key) in all_rules:
rule = all_rules[key](
generate_id(prefix="Rule_"),
pattern,
recipes[pattern.recipe]
)
rules[rule.name] = rule
return rules