refactored monitor handler interaction to better allow differing event types in same system

This commit is contained in:
PatchOfScotland
2023-01-13 18:04:50 +01:00
parent e9519d718f
commit d9004394c1
11 changed files with 357 additions and 355 deletions

View File

@ -6,25 +6,14 @@ from multiprocessing import Pipe, Queue
from time import sleep
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
BAREBONES_NOTEBOOK, SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK
from core.functionality import create_rules, generate_id, wait, \
check_pattern_dict, check_recipe_dict, get_file_hash, rmtree, make_dir, \
parameterize_jupyter_notebook
BAREBONES_NOTEBOOK, SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \
EVENT_TYPE
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
make_dir, parameterize_jupyter_notebook, create_event
from core.meow import BaseRule
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
valid_pattern_one = FileEventPattern(
"pattern_one", "path_one", "recipe_one", "file_one")
valid_pattern_two = FileEventPattern(
"pattern_two", "path_two", "recipe_two", "file_two")
valid_recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
valid_recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
@ -35,9 +24,6 @@ class CorrectnessTests(unittest.TestCase):
super().tearDown()
rmtree(TEST_MONITOR_BASE)
def testCreateRulesMinimum(self)->None:
create_rules({}, {})
def testGenerateIDWorking(self)->None:
id = generate_id()
self.assertEqual(len(id), 16)
@ -64,97 +50,6 @@ class CorrectnessTests(unittest.TestCase):
prefix_id = generate_id(prefix="Test")
self.assertEqual(len(prefix_id), 16)
self.assertTrue(prefix_id.startswith("Test"))
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,
valid_pattern_two.name: valid_pattern_two
}
recipes = {
valid_recipe_one.name: valid_recipe_one,
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
self.assertIsInstance(rule, BaseRule)
self.assertEqual(k, rule.name)
def testCreateRulesMisindexedPatterns(self)->None:
patterns = {
valid_pattern_two.name: valid_pattern_one,
valid_pattern_one.name: valid_pattern_two
}
with self.assertRaises(KeyError):
create_rules(patterns, {})
def testCreateRulesMisindexedRecipes(self)->None:
recipes = {
valid_recipe_two.name: valid_recipe_one,
valid_recipe_one.name: valid_recipe_two
}
with self.assertRaises(KeyError):
create_rules({}, recipes)
def testCheckPatternDictValid(self)->None:
fep1 = FileEventPattern("name_one", "path", "recipe", "file")
fep2 = FileEventPattern("name_two", "path", "recipe", "file")
patterns = {
fep1.name: fep1,
fep2.name: fep2
}
check_pattern_dict(patterns=patterns)
def testCheckPatternDictNoEntries(self)->None:
with self.assertRaises(ValueError):
check_pattern_dict(patterns={})
check_pattern_dict(patterns={}, min_length=0)
def testCheckPatternDictMissmatchedName(self)->None:
fep1 = FileEventPattern("name_one", "path", "recipe", "file")
fep2 = FileEventPattern("name_two", "path", "recipe", "file")
patterns = {
fep2.name: fep1,
fep1.name: fep2
}
with self.assertRaises(KeyError):
check_pattern_dict(patterns=patterns)
def testCheckRecipeDictValid(self)->None:
jnr1 = JupyterNotebookRecipe("recipe_one", BAREBONES_NOTEBOOK)
jnr2 = JupyterNotebookRecipe("recipe_two", BAREBONES_NOTEBOOK)
recipes = {
jnr1.name: jnr1,
jnr2.name: jnr2
}
check_recipe_dict(recipes=recipes)
def testCheckRecipeDictNoEntires(self)->None:
with self.assertRaises(ValueError):
check_recipe_dict(recipes={})
check_recipe_dict(recipes={}, min_length=0)
def testCheckRecipeDictMismatchedName(self)->None:
jnr1 = JupyterNotebookRecipe("recipe_one", BAREBONES_NOTEBOOK)
jnr2 = JupyterNotebookRecipe("recipe_two", BAREBONES_NOTEBOOK)
recipes = {
jnr2.name: jnr1,
jnr1.name: jnr2
}
with self.assertRaises(KeyError):
check_recipe_dict(recipes=recipes)
def testWaitPipes(self)->None:
pipe_one_reader, pipe_one_writer = Pipe()
@ -326,3 +221,20 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual(
pn["cells"][0]["source"],
"# The first cell\n\ns = 4\nnum = 1000")
def testCreateEvent(self)->None:
event = create_event("test")
self.assertEqual(type(event), dict)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertEqual(len(event.keys()), 1)
self.assertEqual(event[EVENT_TYPE], "test")
event2 = create_event("test2", {"a":1})
self.assertEqual(type(event2), dict)
self.assertTrue(EVENT_TYPE in event2.keys())
self.assertEqual(len(event2.keys()), 2)
self.assertEqual(event2[EVENT_TYPE], "test2")
self.assertEqual(event2["a"], 1)

View File

@ -8,14 +8,24 @@ from time import sleep
from typing import Any
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
TEST_MONITOR_BASE, APPENDING_NOTEBOOK
from core.functionality import make_dir, rmtree, create_rules, read_notebook
TEST_MONITOR_BASE, APPENDING_NOTEBOOK, BAREBONES_NOTEBOOK
from core.functionality import make_dir, rmtree, read_notebook
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, MeowRunner
BaseHandler, MeowRunner, create_rules
from patterns import WatchdogMonitor, FileEventPattern
from recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe, RESULT_FILE
valid_pattern_one = FileEventPattern(
"pattern_one", "path_one", "recipe_one", "file_one")
valid_pattern_two = FileEventPattern(
"pattern_two", "path_two", "recipe_two", "file_two")
valid_recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
valid_recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
class MeowTests(unittest.TestCase):
def setUp(self) -> None:
@ -84,39 +94,75 @@ class MeowTests(unittest.TestCase):
pass
FullRule("name", "", "")
def testCreateRulesMinimum(self)->None:
create_rules({}, {})
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,
valid_pattern_two.name: valid_pattern_two
}
recipes = {
valid_recipe_one.name: valid_recipe_one,
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
self.assertIsInstance(rule, BaseRule)
self.assertEqual(k, rule.name)
def testCreateRulesMisindexedPatterns(self)->None:
patterns = {
valid_pattern_two.name: valid_pattern_one,
valid_pattern_one.name: valid_pattern_two
}
with self.assertRaises(KeyError):
create_rules(patterns, {})
def testCreateRulesMisindexedRecipes(self)->None:
recipes = {
valid_recipe_two.name: valid_recipe_one,
valid_recipe_one.name: valid_recipe_two
}
with self.assertRaises(KeyError):
create_rules({}, recipes)
def testBaseMonitor(self)->None:
with self.assertRaises(TypeError):
BaseMonitor("", "")
BaseMonitor("")
class TestMonitor(BaseMonitor):
pass
with self.assertRaises(NotImplementedError):
TestMonitor("", "")
TestMonitor("")
class FullTestMonitor(BaseMonitor):
def start(self):
pass
def stop(self):
pass
def _is_valid_report(self, report:Any)->None:
def _is_valid_to_runner(self, to_runner:Any)->None:
pass
def _is_valid_rules(self, rules:Any)->None:
pass
FullTestMonitor("", "")
FullTestMonitor("")
def testBaseHandler(self)->None:
with self.assertRaises(TypeError):
BaseHandler("")
BaseHandler()
class TestHandler(BaseHandler):
pass
with self.assertRaises(NotImplementedError):
TestHandler("")
TestHandler()
class FullTestHandler(BaseHandler):
def handle(self, event, rule):
def handle(self, event):
pass
def start(self):
pass
@ -124,10 +170,12 @@ class MeowTests(unittest.TestCase):
pass
def _is_valid_inputs(self, inputs:Any)->None:
pass
FullTestHandler("")
def valid_event_types(self)->list[str]:
pass
FullTestHandler()
def testMeowRunner(self)->None:
monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
#monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
@ -153,12 +201,11 @@ class MeowTests(unittest.TestCase):
WatchdogMonitor(
TEST_MONITOR_BASE,
rules,
monitor_to_handler_writer,
print=monitor_debug_stream,
logging=3, settletime=1
logging=3,
settletime=1
),
PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=handler_debug_stream,
@ -213,7 +260,7 @@ class MeowTests(unittest.TestCase):
self.assertEqual(data, "Initial Data\nA line from a test Pattern")
def testMeowRunnerLinkeExecution(self)->None:
monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
#monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
@ -246,12 +293,11 @@ class MeowTests(unittest.TestCase):
WatchdogMonitor(
TEST_MONITOR_BASE,
rules,
monitor_to_handler_writer,
print=monitor_debug_stream,
logging=3, settletime=1
logging=3,
settletime=1
),
PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=handler_debug_stream,

View File

@ -5,8 +5,10 @@ import unittest
from multiprocessing import Pipe
from core.correctness.vars import FILE_EVENTS, FILE_CREATE_EVENT, \
BAREBONES_NOTEBOOK, TEST_MONITOR_BASE
from core.functionality import create_rules, rmtree, make_dir
BAREBONES_NOTEBOOK, TEST_MONITOR_BASE, EVENT_TYPE, WATCHDOG_RULE, \
WATCHDOG_BASE, WATCHDOG_SRC, WATCHDOG_TYPE
from core.functionality import rmtree, make_dir
from core.meow import create_rules
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
_DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP
from recipes import JupyterNotebookRecipe
@ -143,7 +145,6 @@ class CorrectnessTests(unittest.TestCase):
fep = FileEventPattern("name", "path", "recipe", "file",
sweep=bad_sweep)
def testWatchdogMonitorMinimum(self)->None:
from_monitor = Pipe()
WatchdogMonitor(TEST_MONITOR_BASE, {}, from_monitor[1])
@ -164,8 +165,13 @@ class CorrectnessTests(unittest.TestCase):
}
rules = create_rules(patterns, recipes)
wm = WatchdogMonitor(TEST_MONITOR_BASE, rules, from_monitor_writer)
wm = WatchdogMonitor(TEST_MONITOR_BASE, rules)
wm.to_runner = from_monitor_writer
self.assertEqual(len(rules), 1)
rule = rules[list(rules.keys())[0]]
# TODO fix this test
wm.start()
open(os.path.join(TEST_MONITOR_BASE, "A"), "w")
@ -173,10 +179,17 @@ class CorrectnessTests(unittest.TestCase):
message = from_monitor_reader.recv()
self.assertIsNotNone(message)
event, rule = message
event = message
self.assertIsNotNone(event)
self.assertIsNotNone(rule)
self.assertEqual(event.src_path, os.path.join(TEST_MONITOR_BASE, "A"))
self.assertEqual(type(event), dict)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(WATCHDOG_SRC in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertTrue(WATCHDOG_RULE in event.keys())
self.assertEqual(event[EVENT_TYPE], WATCHDOG_TYPE)
self.assertEqual(event[WATCHDOG_SRC], os.path.join(TEST_MONITOR_BASE, "A"))
self.assertEqual(event[WATCHDOG_BASE], TEST_MONITOR_BASE)
self.assertEqual(event[WATCHDOG_RULE].name, rule.name)
open(os.path.join(TEST_MONITOR_BASE, "B"), "w")
if from_monitor_reader.poll(3):

View File

@ -13,8 +13,10 @@ from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, BASE_FILE, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from core.correctness.vars import BAREBONES_NOTEBOOK, TEST_HANDLER_BASE, \
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK
from core.functionality import rmtree, make_dir, create_rules, read_notebook
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, \
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_SRC, WATCHDOG_TYPE
from core.functionality import rmtree, make_dir, read_notebook
from core.meow import create_rules
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
@ -92,59 +94,15 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual(jnr.source, source)
def testPapermillHanderMinimum(self)->None:
monitor_to_handler_reader, _ = Pipe()
PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
def testPapermillHanderStartStop(self)->None:
monitor_to_handler_reader, _ = Pipe()
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.start()
ph.stop()
def testPapermillHanderRepeatedStarts(self)->None:
monitor_to_handler_reader, _ = Pipe()
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.start()
with self.assertRaises(RuntimeWarning):
ph.start()
ph.stop()
def testPapermillHanderStopBeforeStart(self)->None:
monitor_to_handler_reader, _ = Pipe()
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
with self.assertRaises(RuntimeWarning):
ph.stop()
def testPapermillHandlerHandling(self)->None:
monitor_to_handler_reader, to_handler = Pipe()
debug_stream = io.StringIO("")
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=debug_stream,
@ -153,8 +111,6 @@ class CorrectnessTests(unittest.TestCase):
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
event = FileCreatedEvent(os.path.join(TEST_MONITOR_BASE, "A"))
event.monitor_base = TEST_MONITOR_BASE
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
@ -175,8 +131,14 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
ph.start()
to_handler.send((event, rule))
event = {
EVENT_TYPE: WATCHDOG_TYPE,
WATCHDOG_SRC: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule
}
ph.handle(event)
loops = 0
job_id = None
@ -211,5 +173,3 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual("124875.0\n",
result["cells"][4]["outputs"][0]["text"][0])
ph.stop()

View File

@ -6,8 +6,9 @@ from typing import Any, Union
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_existing_dir_path, valid_non_existing_path
from core.correctness.vars import VALID_NAME_CHARS, TEST_MONITOR_BASE, SHA256
valid_existing_dir_path, valid_non_existing_path, valid_event
from core.correctness.vars import VALID_NAME_CHARS, TEST_MONITOR_BASE, \
SHA256, EVENT_TYPE
from core.functionality import rmtree, make_dir
class CorrectnessTests(unittest.TestCase):
@ -18,6 +19,7 @@ class CorrectnessTests(unittest.TestCase):
def tearDown(self) -> None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree("first")
def testCheckTypeValid(self)->None:
check_type(1, int)
@ -204,3 +206,15 @@ class CorrectnessTests(unittest.TestCase):
make_dir("first/second")
with self.assertRaises(ValueError):
valid_non_existing_path("first/second")
def testEventValidation(self)->None:
valid_event({EVENT_TYPE: "test"})
valid_event({EVENT_TYPE: "another"})
valid_event({EVENT_TYPE: "anything", "a": 1})
valid_event({EVENT_TYPE: "something", 1: 1})
with self.assertRaises(KeyError):
valid_event({"EVENT_TYPE": "test"})
with self.assertRaises(KeyError):
valid_event({})