diff --git a/core/meow.py b/core/meow.py index 9bc68a3..8c4f629 100644 --- a/core/meow.py +++ b/core/meow.py @@ -130,14 +130,29 @@ class BaseRule: class BaseMonitor: - rules: dict[str, BaseRule] + _patterns: dict[str, BasePattern] + _recipes: dict[str, BaseRecipe] + _rules: dict[str, BaseRule] to_runner: VALID_CHANNELS - def __init__(self, rules:dict[str,BaseRule])->None: + def __init__(self, patterns:dict[str,BasePattern], recipes:dict[str,BaseRecipe])->None: check_implementation(type(self).start, BaseMonitor) check_implementation(type(self).stop, BaseMonitor) - check_implementation(type(self)._is_valid_rules, BaseMonitor) - self._is_valid_rules(rules) - self.rules = rules + check_implementation(type(self)._is_valid_patterns, BaseMonitor) + self._is_valid_patterns(patterns) + check_implementation(type(self)._is_valid_recipes, BaseMonitor) + self._is_valid_recipes(recipes) + check_implementation(type(self).add_pattern, BaseMonitor) + check_implementation(type(self).update_pattern, BaseMonitor) + check_implementation(type(self).remove_pattern, BaseMonitor) + check_implementation(type(self).get_patterns, BaseMonitor) + check_implementation(type(self).add_recipe, BaseMonitor) + check_implementation(type(self).update_recipe, BaseMonitor) + check_implementation(type(self).remove_recipe, BaseMonitor) + check_implementation(type(self).get_recipes, BaseMonitor) + check_implementation(type(self).get_rules, BaseMonitor) + self._patterns = patterns + self._recipes = recipes + self._rules = create_rules(patterns, recipes) def __new__(cls, *args, **kwargs): if cls is BaseMonitor: @@ -145,7 +160,10 @@ class BaseMonitor: raise TypeError(msg) return object.__new__(cls) - def _is_valid_rules(self, rules:dict[str,BaseRule])->None: + def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None: + pass + + def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None: pass def start(self)->None: @@ -154,6 +172,33 @@ class BaseMonitor: def stop(self)->None: pass + def add_pattern(self, pattern:BasePattern)->None: + pass + + def update_pattern(self, pattern:BasePattern)->None: + pass + + def remove_pattern(self, pattern:Union[str,BasePattern])->None: + pass + + def get_patterns(self)->None: + pass + + def add_recipe(self, recipe:BaseRecipe)->None: + pass + + def update_recipe(self, recipe:BaseRecipe)->None: + pass + + def remove_recipe(self, recipe:Union[str,BaseRecipe])->None: + pass + + def get_recipes(self)->None: + pass + + def get_rules(self)->None: + pass + class BaseHandler: def __init__(self) -> None: @@ -172,7 +217,6 @@ class BaseHandler: def handle(self, event:Any)->None: pass - def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]], recipes:Union[dict[str,BaseRecipe],list[BaseRecipe]], new_rules:list[BaseRule]=[])->dict[str,BaseRule]: @@ -204,22 +248,31 @@ def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]], "Recipe dictionaries must be keyed with the name of the " "Recipe.") + generated_rules = {} + for pattern in patterns.values(): + if pattern.recipe in recipes: + rule = create_rule(pattern, recipes[pattern.recipe]) + generated_rules[rule.name] = rule + return generated_rules + +def create_rule(pattern:BasePattern, recipe:BaseRecipe, + new_rules:list[BaseRule]=[])->BaseRule: + check_type(pattern, BasePattern) + check_type(recipe, BaseRecipe) + valid_list(new_rules, BaseRule, min_length=0) + # Imported here to avoid circular imports at top of file import rules - rules = {} all_rules ={(r.pattern_type, r.recipe_type):r for r in [r[1] \ for r in inspect.getmembers(sys.modules["rules"], inspect.isclass) \ if (issubclass(r[1], BaseRule))]} - for pattern in patterns.values(): - if pattern.recipe in recipes: - key = (type(pattern).__name__, - type(recipes[pattern.recipe]).__name__) - if (key) in all_rules: - rule = all_rules[key]( - generate_id(prefix="Rule_"), - pattern, - recipes[pattern.recipe] - ) - rules[rule.name] = rule - return rules + key = (type(pattern).__name__, type(recipe).__name__) + if (key) in all_rules: + return all_rules[key]( + generate_id(prefix="Rule_"), + pattern, + recipe + ) + raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe " + f"'{recipe}' could be found.") diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 5e170f9..7cbaec8 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -4,10 +4,11 @@ import threading import sys import os +from copy import deepcopy from fnmatch import translate from re import match from time import time, sleep -from typing import Any +from typing import Any, Union from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler @@ -19,7 +20,8 @@ from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, WATCHDOG_TYPE, \ WATCHDOG_SRC, WATCHDOG_RULE, WATCHDOG_BASE, FILE_RETROACTIVE_EVENT from core.functionality import print_debug, create_event -from core.meow import BasePattern, BaseMonitor, BaseRule +from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \ + create_rule _DEFAULT_MASK = [ FILE_CREATE_EVENT, @@ -124,16 +126,20 @@ class WatchdogMonitor(BaseMonitor): base_dir:str debug_level:int _print_target:Any + _patterns_lock:threading.Lock + _recipes_lock:threading.Lock _rules_lock:threading.Lock - def __init__(self, base_dir:str, rules:dict[str, BaseRule], - autostart=False, settletime:int=1, print:Any=sys.stdout, - logging:int=0)->None: - super().__init__(rules) + def __init__(self, base_dir:str, patterns:dict[str,FileEventPattern], + recipes:dict[str,BaseRecipe], autostart=False, settletime:int=1, + print:Any=sys.stdout, logging:int=0)->None: + super().__init__(patterns, recipes) self._is_valid_base_dir(base_dir) self.base_dir = base_dir check_type(settletime, int) self._print_target, self.debug_level = setup_debugging(print, logging) + self._patterns_lock = threading.Lock() + self._recipes_lock = threading.Lock() self._rules_lock = threading.Lock() self.event_handler = WatchdogEventHandler(self, settletime=settletime) self.monitor = Observer() @@ -170,7 +176,7 @@ class WatchdogMonitor(BaseMonitor): self._rules_lock.acquire() try: - for rule in self.rules.values(): + for rule in self._rules.values(): if event_type not in rule.pattern.event_mask: continue @@ -199,19 +205,210 @@ class WatchdogMonitor(BaseMonitor): self._rules_lock.release() + def add_pattern(self, pattern: FileEventPattern) -> None: + check_type(pattern, FileEventPattern) + self._patterns_lock.acquire() + try: + if pattern.name in self._patterns: + raise KeyError(f"An entry for Pattern '{pattern.name}' already " + "exists. Do you intend to update instead?") + self._patterns[pattern.name] = pattern + except Exception as e: + self._patterns_lock.release() + raise Exception(e) + self._patterns_lock.release() + + self._identify_new_rules(new_pattern=pattern) + + def update_pattern(self, pattern: FileEventPattern) -> None: + check_type(pattern, FileEventPattern) + self.remove_pattern(pattern.name) + self.add_pattern(pattern) + + def remove_pattern(self, pattern: Union[str, FileEventPattern]) -> None: + check_type(pattern, str, alt_types=[FileEventPattern]) + lookup_key = pattern + if type(lookup_key) is FileEventPattern: + lookup_key = pattern.name + self._patterns_lock.acquire() + try: + if lookup_key not in self._patterns: + raise KeyError(f"Cannot remote Pattern '{lookup_key}' as it " + "does not already exist") + self._patterns.pop(lookup_key) + except Exception as e: + self._patterns_lock.release() + raise Exception(e) + self._patterns_lock.release() + + self._identify_lost_rules(lost_pattern=pattern.name) + + def get_patterns(self) -> None: + to_return = {} + self._patterns_lock.acquire() + try: + to_return = deepcopy(self._patterns) + except Exception as e: + self._patterns_lock.release() + raise Exception(e) + self._patterns_lock.release() + return to_return + + def add_recipe(self, recipe: BaseRecipe) -> None: + check_type(recipe, BaseRecipe) + self._recipes_lock.acquire() + try: + if recipe.name in self._recipes: + raise KeyError(f"An entry for Recipe '{recipe.name}' already " + "exists. Do you intend to update instead?") + self._recipes[recipe.name] = recipe + except Exception as e: + self._recipes_lock.release() + raise Exception(e) + self._recipes_lock.release() + + self._identify_new_rules(new_recipe=recipe) + + def update_recipe(self, recipe: BaseRecipe) -> None: + check_type(recipe, BaseRecipe) + self.remove_recipe(recipe.name) + self.add_recipe(recipe) + + def remove_recipe(self, recipe: Union[str, BaseRecipe]) -> None: + check_type(recipe, str, alt_types=[BaseRecipe]) + lookup_key = recipe + if type(lookup_key) is BaseRecipe: + lookup_key = recipe.name + self._recipes_lock.acquire() + try: + if lookup_key not in self._recipes: + raise KeyError(f"Cannot remote Recipe '{lookup_key}' as it " + "does not already exist") + self._recipes.pop(lookup_key) + except Exception as e: + self._recipes_lock.release() + raise Exception(e) + self._recipes_lock.release() + + self._identify_lost_rules(lost_recipe=recipe.name) + + def get_recipes(self) -> None: + to_return = {} + self._recipes_lock.acquire() + try: + to_return = deepcopy(self._recipes) + except Exception as e: + self._recipes_lock.release() + raise Exception(e) + self._recipes_lock.release() + return to_return + + def get_rules(self) -> None: + to_return = {} + self._rules_lock.acquire() + try: + to_return = deepcopy(self._rules) + except Exception as e: + self._rules_lock.release() + raise Exception(e) + self._rules_lock.release() + return to_return + + def _identify_new_rules(self, new_pattern:FileEventPattern=None, + new_recipe:BaseRecipe=None)->None: + + if new_pattern: + self._patterns_lock.acquire() + self._recipes_lock.acquire() + try: + if new_pattern.name not in self._patterns: + self._patterns_lock.release() + self._recipes_lock.release() + return + if new_pattern.recipe in self._recipes: + self._create_new_rule( + new_pattern, + self._recipes[new_pattern.recipe], + ) + except Exception as e: + self._patterns_lock.release() + self._recipes_lock.release() + raise Exception(e) + self._patterns_lock.release() + self._recipes_lock.release() + + if new_recipe: + self._patterns_lock.acquire() + self._patterns_lock.acquire() + try: + if new_recipe.name not in self._recipes: + self._patterns_lock.release() + self._recipes_lock.release() + return + for pattern in self._patterns.values(): + if pattern.recipe == new_recipe.name: + self._create_new_rule( + pattern, + new_recipe, + ) + except Exception as e: + self._patterns_lock.release() + self._recipes_lock.release() + raise Exception(e) + self._patterns_lock.release() + self._recipes_lock.release() + + def _identify_lost_rules(self, lost_pattern:str, lost_recipe:str)->None: + to_delete = [] + self._rules_lock.acquire() + try: + for name, rule in self._rules.items(): + if lost_pattern and rule.pattern.name == lost_pattern: + to_delete.append(name) + if lost_recipe and rule.recipe.name == lost_recipe: + to_delete.append(name) + for delete in to_delete: + if delete in self._rules.keys(): + self._rules.pop(delete) + except Exception as e: + self._rules_lock.release() + raise Exception(e) + self._rules_lock.release() + + def _create_new_rule(self, pattern:FileEventPattern, recipe:BaseRecipe)->None: + rule = create_rule(pattern, recipe) + self._rules_lock.acquire() + try: + if rule.name in self._rules: + raise KeyError("Cannot create Rule with name of " + f"'{rule.name}' as already in use") + self._rules[rule.name] = rule + except Exception as e: + self._rules_lock.release() + raise Exception(e) + self._rules_lock.release() + + self._apply_retroactive_rule(rule) + def _is_valid_base_dir(self, base_dir:str)->None: valid_existing_dir_path(base_dir) - def _is_valid_rules(self, rules:dict[str, BaseRule])->None: - valid_dict(rules, str, BaseRule, min_length=0, strict=False) + def _is_valid_patterns(self, patterns:dict[str,FileEventPattern])->None: + valid_dict(patterns, str, FileEventPattern, min_length=0, strict=False) + + def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None: + valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False) def _apply_retroactive_rules(self)->None: - for rule in self.rules.values(): + for rule in self._rules.values(): self._apply_retroactive_rule(rule) def _apply_retroactive_rule(self, rule:BaseRule)->None: self._rules_lock.acquire() try: + if rule.name not in self._rules: + self._rules_lock.release() + return if FILE_RETROACTIVE_EVENT in rule.pattern.event_mask: testing_path = os.path.join(self.base_dir, rule.pattern.triggering_path) diff --git a/tests/test_meow.py b/tests/test_meow.py index bcbb460..16fc9b6 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -4,7 +4,7 @@ import os import unittest from multiprocessing import Pipe -from typing import Any +from typing import Any, Union from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \ TEST_MONITOR_BASE, BAREBONES_NOTEBOOK, WATCHDOG_BASE, WATCHDOG_RULE, \ @@ -131,24 +131,42 @@ class MeowTests(unittest.TestCase): def testBaseMonitor(self)->None: with self.assertRaises(TypeError): - BaseMonitor("") + BaseMonitor({}, {}) class TestMonitor(BaseMonitor): pass with self.assertRaises(NotImplementedError): - TestMonitor("") + TestMonitor({}, {}) class FullTestMonitor(BaseMonitor): def start(self): pass def stop(self): pass - def _is_valid_to_runner(self, to_runner:Any)->None: + def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None: pass - def _is_valid_rules(self, rules:Any)->None: + def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None: pass - FullTestMonitor("") + def add_pattern(self, pattern:BasePattern)->None: + pass + def update_pattern(self, pattern:BasePattern)->None: + pass + def remove_pattern(self, pattern:Union[str,BasePattern])->None: + pass + def get_patterns(self)->None: + pass + def add_recipe(self, recipe:BaseRecipe)->None: + pass + def update_recipe(self, recipe:BaseRecipe)->None: + pass + def remove_recipe(self, recipe:Union[str,BaseRecipe])->None: + pass + def get_recipes(self)->None: + pass + def get_rules(self)->None: + pass + FullTestMonitor({}, {}) def testMonitoring(self)->None: pattern_one = FileEventPattern( @@ -163,20 +181,21 @@ class MeowTests(unittest.TestCase): recipes = { recipe.name: recipe, } - rules = create_rules(patterns, recipes) - - rule = rules[list(rules.keys())[0]] monitor_debug_stream = io.StringIO("") wm = WatchdogMonitor( TEST_MONITOR_BASE, - rules, + patterns, + recipes, print=monitor_debug_stream, logging=3, settletime=1 ) + rules = wm.get_rules() + rule = rules[list(rules.keys())[0]] + from_monitor_reader, from_monitor_writer = Pipe() wm.to_runner = from_monitor_writer @@ -225,9 +244,6 @@ class MeowTests(unittest.TestCase): recipes = { recipe.name: recipe, } - rules = create_rules(patterns, recipes) - - rule = rules[list(rules.keys())[0]] start_dir = os.path.join(TEST_MONITOR_BASE, "start") make_dir(start_dir) @@ -241,12 +257,16 @@ class MeowTests(unittest.TestCase): wm = WatchdogMonitor( TEST_MONITOR_BASE, - rules, + patterns, + recipes, print=monitor_debug_stream, logging=3, settletime=1 ) + rules = wm.get_rules() + rule = rules[list(rules.keys())[0]] + from_monitor_reader, from_monitor_writer = Pipe() wm.to_runner = from_monitor_writer diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 907c75a..ba8e8c5 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -147,7 +147,7 @@ class CorrectnessTests(unittest.TestCase): def testWatchdogMonitorMinimum(self)->None: from_monitor = Pipe() - WatchdogMonitor(TEST_MONITOR_BASE, {}, from_monitor[1]) + WatchdogMonitor(TEST_MONITOR_BASE, {}, {}, from_monitor[1]) def testWatchdogMonitorEventIdentificaion(self)->None: from_monitor_reader, from_monitor_writer = Pipe() @@ -163,11 +163,12 @@ class CorrectnessTests(unittest.TestCase): recipes = { recipe.name: recipe, } - rules = create_rules(patterns, recipes) - wm = WatchdogMonitor(TEST_MONITOR_BASE, rules) + wm = WatchdogMonitor(TEST_MONITOR_BASE, patterns, recipes) wm.to_runner = from_monitor_writer - + + rules = wm.get_rules() + self.assertEqual(len(rules), 1) rule = rules[list(rules.keys())[0]] diff --git a/tests/test_runner.py b/tests/test_runner.py index 2e26076..3327ab2 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -44,7 +44,6 @@ class MeowTests(unittest.TestCase): recipes = { recipe.name: recipe, } - rules = create_rules(patterns, recipes) monitor_debug_stream = io.StringIO("") handler_debug_stream = io.StringIO("") @@ -52,7 +51,8 @@ class MeowTests(unittest.TestCase): runner = MeowRunner( WatchdogMonitor( TEST_MONITOR_BASE, - rules, + patterns, + recipes, print=monitor_debug_stream, logging=3, settletime=1 @@ -142,7 +142,8 @@ class MeowTests(unittest.TestCase): runner = MeowRunner( WatchdogMonitor( TEST_MONITOR_BASE, - rules, + patterns, + recipes, print=monitor_debug_stream, logging=3, settletime=1