diff --git a/core/correctness/vars.py b/core/correctness/vars.py index 87ab2af..2682ab6 100644 --- a/core/correctness/vars.py +++ b/core/correctness/vars.py @@ -200,23 +200,27 @@ FILE_MODIFY_EVENT = "file_modified" FILE_MOVED_EVENT = "file_moved" FILE_CLOSED_EVENT = "file_closed" FILE_DELETED_EVENT = "file_deleted" +FILE_RETROACTIVE_EVENT = "retroactive_file_event" FILE_EVENTS = [ FILE_CREATE_EVENT, FILE_MODIFY_EVENT, FILE_MOVED_EVENT, FILE_CLOSED_EVENT, - FILE_DELETED_EVENT + FILE_DELETED_EVENT, + FILE_RETROACTIVE_EVENT ] DIR_CREATE_EVENT = "dir_created" DIR_MODIFY_EVENT = "dir_modified" DIR_MOVED_EVENT = "dir_moved" DIR_DELETED_EVENT = "dir_deleted" +DIR_RETROACTIVE_EVENT = "retroactive_dir_event" DIR_EVENTS = [ DIR_CREATE_EVENT, DIR_MODIFY_EVENT, DIR_MOVED_EVENT, - DIR_DELETED_EVENT + DIR_DELETED_EVENT, + DIR_RETROACTIVE_EVENT ] # debug printing levels diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 049a9e9..5e170f9 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -1,4 +1,5 @@ +import glob import threading import sys import os @@ -15,16 +16,16 @@ from core.correctness.validation import check_type, valid_string, \ setup_debugging from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \ - FILE_MODIFY_EVENT, FILE_MOVED_EVENT, VALID_CHANNELS, DEBUG_INFO, \ - DEBUG_ERROR, DEBUG_WARNING, WATCHDOG_TYPE, WATCHDOG_SRC, WATCHDOG_RULE, \ - WATCHDOG_BASE + FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, WATCHDOG_TYPE, \ + WATCHDOG_SRC, WATCHDOG_RULE, WATCHDOG_BASE, FILE_RETROACTIVE_EVENT from core.functionality import print_debug, create_event from core.meow import BasePattern, BaseMonitor, BaseRule _DEFAULT_MASK = [ FILE_CREATE_EVENT, FILE_MODIFY_EVENT, - FILE_MOVED_EVENT + FILE_MOVED_EVENT, + FILE_RETROACTIVE_EVENT ] SWEEP_START = "start" @@ -150,6 +151,7 @@ class WatchdogMonitor(BaseMonitor): def start(self)->None: print_debug(self._print_target, self.debug_level, "Starting WatchdogMonitor", DEBUG_INFO) + self._apply_retroactive_rules() self.monitor.start() def stop(self)->None: @@ -203,6 +205,36 @@ class WatchdogMonitor(BaseMonitor): def _is_valid_rules(self, rules:dict[str, BaseRule])->None: valid_dict(rules, str, BaseRule, min_length=0, strict=False) + def _apply_retroactive_rules(self)->None: + for rule in self.rules.values(): + self._apply_retroactive_rule(rule) + + def _apply_retroactive_rule(self, rule:BaseRule)->None: + self._rules_lock.acquire() + try: + if FILE_RETROACTIVE_EVENT in rule.pattern.event_mask: + + testing_path = os.path.join(self.base_dir, rule.pattern.triggering_path) + + globbed = glob.glob(testing_path) + + for globble in globbed: + + meow_event = create_event( + WATCHDOG_TYPE, { + WATCHDOG_SRC: globble, + WATCHDOG_BASE: self.base_dir, + WATCHDOG_RULE: rule + }) + print_debug(self._print_target, self.debug_level, + f"Retroactive event for file at at {globble} hit rule " + f"{rule.name}", DEBUG_INFO) + self.to_runner.send(meow_event) + + except Exception as e: + self._rules_lock.release() + raise Exception(e) + self._rules_lock.release() class WatchdogEventHandler(PatternMatchingEventHandler): monitor:WatchdogMonitor diff --git a/tests/test_meow.py b/tests/test_meow.py index a31ee49..bcbb460 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -1,14 +1,18 @@ +import io +import os import unittest +from multiprocessing import Pipe from typing import Any from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \ - TEST_MONITOR_BASE, BAREBONES_NOTEBOOK + TEST_MONITOR_BASE, BAREBONES_NOTEBOOK, WATCHDOG_BASE, WATCHDOG_RULE, \ + WATCHDOG_SRC, WATCHDOG_TYPE, EVENT_TYPE from core.functionality import make_dir, rmtree from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \ BaseHandler, create_rules -from patterns import FileEventPattern +from patterns import FileEventPattern, WatchdogMonitor from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe valid_pattern_one = FileEventPattern( @@ -146,6 +150,131 @@ class MeowTests(unittest.TestCase): pass FullTestMonitor("") + def testMonitoring(self)->None: + pattern_one = FileEventPattern( + "pattern_one", "start/A.txt", "recipe_one", "infile", + parameters={}) + recipe = JupyterNotebookRecipe( + "recipe_one", BAREBONES_NOTEBOOK) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + rules = create_rules(patterns, recipes) + + rule = rules[list(rules.keys())[0]] + + monitor_debug_stream = io.StringIO("") + + wm = WatchdogMonitor( + TEST_MONITOR_BASE, + rules, + print=monitor_debug_stream, + logging=3, + settletime=1 + ) + + from_monitor_reader, from_monitor_writer = Pipe() + wm.to_runner = from_monitor_writer + + wm.start() + + start_dir = os.path.join(TEST_MONITOR_BASE, "start") + make_dir(start_dir) + self.assertTrue(start_dir) + with open(os.path.join(start_dir, "A.txt"), "w") as f: + f.write("Initial Data") + + self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt"))) + + messages = [] + while True: + if from_monitor_reader.poll(3): + messages.append(from_monitor_reader.recv()) + else: + break + self.assertTrue(len(messages), 1) + message = messages[0] + + self.assertEqual(type(message), dict) + self.assertIn(EVENT_TYPE, message) + self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE) + self.assertIn(WATCHDOG_BASE, message) + self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE) + self.assertIn(WATCHDOG_SRC, message) + self.assertEqual(message[WATCHDOG_SRC], + os.path.join(start_dir, "A.txt")) + self.assertIn(WATCHDOG_RULE, message) + self.assertEqual(message[WATCHDOG_RULE].name, rule.name) + + wm.stop() + + def testMonitoringRetroActive(self)->None: + pattern_one = FileEventPattern( + "pattern_one", "start/A.txt", "recipe_one", "infile", + parameters={}) + recipe = JupyterNotebookRecipe( + "recipe_one", BAREBONES_NOTEBOOK) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + rules = create_rules(patterns, recipes) + + rule = rules[list(rules.keys())[0]] + + start_dir = os.path.join(TEST_MONITOR_BASE, "start") + make_dir(start_dir) + self.assertTrue(start_dir) + with open(os.path.join(start_dir, "A.txt"), "w") as f: + f.write("Initial Data") + + self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt"))) + + monitor_debug_stream = io.StringIO("") + + wm = WatchdogMonitor( + TEST_MONITOR_BASE, + rules, + print=monitor_debug_stream, + logging=3, + settletime=1 + ) + + from_monitor_reader, from_monitor_writer = Pipe() + wm.to_runner = from_monitor_writer + + wm.start() + + messages = [] + while True: + if from_monitor_reader.poll(3): + messages.append(from_monitor_reader.recv()) + else: + break + self.assertTrue(len(messages), 1) + message = messages[0] + + self.assertEqual(type(message), dict) + self.assertIn(EVENT_TYPE, message) + self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE) + self.assertIn(WATCHDOG_BASE, message) + self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE) + self.assertIn(WATCHDOG_SRC, message) + self.assertEqual(message[WATCHDOG_SRC], + os.path.join(start_dir, "A.txt")) + self.assertIn(WATCHDOG_RULE, message) + self.assertEqual(message[WATCHDOG_RULE].name, rule.name) + + wm.stop() + + def testBaseHandler(self)->None: with self.assertRaises(TypeError): BaseHandler() @@ -168,3 +297,4 @@ class MeowTests(unittest.TestCase): def valid_event_types(self)->list[str]: pass FullTestHandler() + diff --git a/tests/test_runner.py b/tests/test_runner.py index 3b01cd7..2e26076 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -219,4 +219,3 @@ class MeowTests(unittest.TestCase): self.assertEqual(data, "Initial Data\nA line from Pattern 1\nA line from Pattern 2") -