NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added support for retroactive event recognition

Browse Source
This commit is contained in:
PatchOfScotland
2023-01-15 10:57:44 +01:00
parent eabedb4747
commit 6ab9c84745
4 changed files with 174 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
core/correctness/vars.py
View File

@ -200,23 +200,27 @@ FILE_MODIFY_EVENT = "file_modified"
FILE_MOVED_EVENT = "file_moved"
FILE_CLOSED_EVENT = "file_closed"
FILE_DELETED_EVENT = "file_deleted"
FILE_RETROACTIVE_EVENT = "retroactive_file_event"
FILE_EVENTS = [
FILE_CREATE_EVENT,
FILE_MODIFY_EVENT,
FILE_MOVED_EVENT,
FILE_CLOSED_EVENT,
FILE_DELETED_EVENT
FILE_DELETED_EVENT,
FILE_RETROACTIVE_EVENT
]
DIR_CREATE_EVENT = "dir_created"
DIR_MODIFY_EVENT = "dir_modified"
DIR_MOVED_EVENT = "dir_moved"
DIR_DELETED_EVENT = "dir_deleted"
DIR_RETROACTIVE_EVENT = "retroactive_dir_event"
DIR_EVENTS = [
DIR_CREATE_EVENT,
DIR_MODIFY_EVENT,
DIR_MOVED_EVENT,
DIR_DELETED_EVENT
DIR_DELETED_EVENT,
DIR_RETROACTIVE_EVENT
]
# debug printing levels

40
patterns/file_event_pattern.py
View File

@ -1,4 +1,5 @@
import glob
import threading
import sys
import os
@ -15,16 +16,16 @@ from core.correctness.validation import check_type, valid_string, \
setup_debugging
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, VALID_CHANNELS, DEBUG_INFO, \
DEBUG_ERROR, DEBUG_WARNING, WATCHDOG_TYPE, WATCHDOG_SRC, WATCHDOG_RULE, \
WATCHDOG_BASE
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, WATCHDOG_TYPE, \
WATCHDOG_SRC, WATCHDOG_RULE, WATCHDOG_BASE, FILE_RETROACTIVE_EVENT
from core.functionality import print_debug, create_event
from core.meow import BasePattern, BaseMonitor, BaseRule
_DEFAULT_MASK = [
FILE_CREATE_EVENT,
FILE_MODIFY_EVENT,
FILE_MOVED_EVENT
FILE_MOVED_EVENT,
FILE_RETROACTIVE_EVENT
]
SWEEP_START = "start"
@ -150,6 +151,7 @@ class WatchdogMonitor(BaseMonitor):
def start(self)->None:
print_debug(self._print_target, self.debug_level,
"Starting WatchdogMonitor", DEBUG_INFO)
self._apply_retroactive_rules()
self.monitor.start()
def stop(self)->None:
@ -203,6 +205,36 @@ class WatchdogMonitor(BaseMonitor):
def _is_valid_rules(self, rules:dict[str, BaseRule])->None:
valid_dict(rules, str, BaseRule, min_length=0, strict=False)
def _apply_retroactive_rules(self)->None:
for rule in self.rules.values():
self._apply_retroactive_rule(rule)
def _apply_retroactive_rule(self, rule:BaseRule)->None:
self._rules_lock.acquire()
try:
if FILE_RETROACTIVE_EVENT in rule.pattern.event_mask:
testing_path = os.path.join(self.base_dir, rule.pattern.triggering_path)
globbed = glob.glob(testing_path)
for globble in globbed:
meow_event = create_event(
WATCHDOG_TYPE, {
WATCHDOG_SRC: globble,
WATCHDOG_BASE: self.base_dir,
WATCHDOG_RULE: rule
})
print_debug(self._print_target, self.debug_level,
f"Retroactive event for file at at {globble} hit rule "
f"{rule.name}", DEBUG_INFO)
self.to_runner.send(meow_event)
except Exception as e:
self._rules_lock.release()
raise Exception(e)
self._rules_lock.release()
class WatchdogEventHandler(PatternMatchingEventHandler):
monitor:WatchdogMonitor

134
tests/test_meow.py
View File

@ -1,14 +1,18 @@
import io
import os
import unittest
from multiprocessing import Pipe
from typing import Any
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
TEST_MONITOR_BASE, BAREBONES_NOTEBOOK
TEST_MONITOR_BASE, BAREBONES_NOTEBOOK, WATCHDOG_BASE, WATCHDOG_RULE, \
WATCHDOG_SRC, WATCHDOG_TYPE, EVENT_TYPE
from core.functionality import make_dir, rmtree
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, create_rules
from patterns import FileEventPattern
from patterns import FileEventPattern, WatchdogMonitor
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
valid_pattern_one = FileEventPattern(
@ -146,6 +150,131 @@ class MeowTests(unittest.TestCase):
pass
FullTestMonitor("")
def testMonitoring(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
rule = rules[list(rules.keys())[0]]
monitor_debug_stream = io.StringIO("")
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
rules,
print=monitor_debug_stream,
logging=3,
settletime=1
)
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.start()
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
messages = []
while True:
if from_monitor_reader.poll(3):
messages.append(from_monitor_reader.recv())
else:
break
self.assertTrue(len(messages), 1)
message = messages[0]
self.assertEqual(type(message), dict)
self.assertIn(EVENT_TYPE, message)
self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE)
self.assertIn(WATCHDOG_BASE, message)
self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE)
self.assertIn(WATCHDOG_SRC, message)
self.assertEqual(message[WATCHDOG_SRC],
os.path.join(start_dir, "A.txt"))
self.assertIn(WATCHDOG_RULE, message)
self.assertEqual(message[WATCHDOG_RULE].name, rule.name)
wm.stop()
def testMonitoringRetroActive(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
rule = rules[list(rules.keys())[0]]
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
monitor_debug_stream = io.StringIO("")
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
rules,
print=monitor_debug_stream,
logging=3,
settletime=1
)
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.start()
messages = []
while True:
if from_monitor_reader.poll(3):
messages.append(from_monitor_reader.recv())
else:
break
self.assertTrue(len(messages), 1)
message = messages[0]
self.assertEqual(type(message), dict)
self.assertIn(EVENT_TYPE, message)
self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE)
self.assertIn(WATCHDOG_BASE, message)
self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE)
self.assertIn(WATCHDOG_SRC, message)
self.assertEqual(message[WATCHDOG_SRC],
os.path.join(start_dir, "A.txt"))
self.assertIn(WATCHDOG_RULE, message)
self.assertEqual(message[WATCHDOG_RULE].name, rule.name)
wm.stop()
def testBaseHandler(self)->None:
with self.assertRaises(TypeError):
BaseHandler()
@ -168,3 +297,4 @@ class MeowTests(unittest.TestCase):
def valid_event_types(self)->list[str]:
pass
FullTestHandler()

1
tests/test_runner.py
View File

@ -219,4 +219,3 @@ class MeowTests(unittest.TestCase):
self.assertEqual(data,
"Initial Data\nA line from Pattern 1\nA line from Pattern 2")