updated runner structure so that handlers and conductors actually pull from queues in the runner. changes to logic in both are extensive, but most individual functinos are unaffected. I've also moved several functions that were part of individual monitor, handler and conductors to the base classes.

This commit is contained in:
PatchOfScotland
2023-04-20 17:08:06 +02:00
parent b87fd43cfd
commit f306d8b6f2
16 changed files with 1589 additions and 964 deletions

View File

@ -1,7 +1,7 @@
import unittest
from typing import Any, Union, Tuple, Dict
from typing import Any, Union, Tuple, Dict, List
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
@ -146,6 +146,7 @@ class BasePatternTests(unittest.TestCase):
self.assertEqual(len(values), 0)
# TODO test for base functions
class BaseMonitorTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -171,32 +172,15 @@ class BaseMonitorTests(unittest.TestCase):
pass
def stop(self):
pass
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
pass
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
pass
def add_pattern(self, pattern:BasePattern)->None:
pass
def update_pattern(self, pattern:BasePattern)->None:
pass
def remove_pattern(self, pattern:Union[str,BasePattern])->None:
pass
def get_patterns(self)->None:
pass
def add_recipe(self, recipe:BaseRecipe)->None:
pass
def update_recipe(self, recipe:BaseRecipe)->None:
pass
def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
pass
def get_recipes(self)->None:
pass
def get_rules(self)->None:
pass
def _get_valid_pattern_types(self)->List[type]:
return [BasePattern]
def _get_valid_recipe_types(self)->List[type]:
return [BaseRecipe]
FullTestMonitor({}, {})
# TODO test for base functions
class BaseHandleTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -220,12 +204,6 @@ class BaseHandleTests(unittest.TestCase):
class FullTestHandler(BaseHandler):
def handle(self, event):
pass
def start(self):
pass
def stop(self):
pass
def _is_valid_inputs(self, inputs:Any)->None:
pass
def valid_handle_criteria(self, event:Dict[str,Any]
)->Tuple[bool,str]:
pass
@ -233,6 +211,7 @@ class BaseHandleTests(unittest.TestCase):
FullTestHandler()
# TODO test for base functions
class BaseConductorTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()

View File

@ -227,7 +227,7 @@ class WatchdogMonitorTests(unittest.TestCase):
}
wm = WatchdogMonitor(TEST_MONITOR_BASE, patterns, recipes)
wm.to_runner = from_monitor_writer
wm.to_runner_event = from_monitor_writer
rules = wm.get_rules()
@ -291,7 +291,7 @@ class WatchdogMonitorTests(unittest.TestCase):
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.to_runner_event = from_monitor_writer
wm.start()
@ -356,7 +356,7 @@ class WatchdogMonitorTests(unittest.TestCase):
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.to_runner_event = from_monitor_writer
wm.start()
@ -437,7 +437,7 @@ class WatchdogMonitorTests(unittest.TestCase):
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.to_runner_event = from_monitor_writer
wm.start()
@ -508,7 +508,7 @@ class WatchdogMonitorTests(unittest.TestCase):
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.to_runner_event = from_monitor_writer
wm.start()

View File

@ -142,9 +142,9 @@ class PapermillHandlerTests(unittest.TestCase):
# Test PapermillHandler will handle given events
def testPapermillHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -180,8 +180,8 @@ class PapermillHandlerTests(unittest.TestCase):
ph.handle(event)
if from_handler_reader.poll(3):
job_dir = from_handler_reader.recv()
if from_handler_to_job_reader.poll(3):
job_dir = from_handler_to_job_reader.recv()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
@ -191,9 +191,9 @@ class PapermillHandlerTests(unittest.TestCase):
# Test PapermillHandler will create enough jobs from single sweep
def testPapermillHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -234,8 +234,8 @@ class PapermillHandlerTests(unittest.TestCase):
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
if from_handler_to_job_reader.poll(3):
jobs.append(from_handler_to_job_reader.recv())
else:
recieving = False
@ -256,9 +256,9 @@ class PapermillHandlerTests(unittest.TestCase):
# Test PapermillHandler will create enough jobs from multiple sweeps
def testPapermillHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -304,8 +304,8 @@ class PapermillHandlerTests(unittest.TestCase):
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
if from_handler_to_job_reader.poll(3):
jobs.append(from_handler_to_job_reader.recv())
else:
recieving = False
@ -477,6 +477,90 @@ class PapermillHandlerTests(unittest.TestCase):
self.assertEqual(recipe.name, "name")
self.assertEqual(recipe.recipe, COMPLETE_NOTEBOOK)
# Test handler starts and stops appropriatly
def testPapermillHandlerStartStop(self)->None:
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
from_handler_to_event_reader, from_handler_to_event_writer = Pipe()
ph.to_runner_event = from_handler_to_event_writer
with self.assertRaises(AttributeError):
self.assertFalse(ph._handle_thread.is_alive())
ph.start()
if from_handler_to_event_reader.poll(3):
msg = from_handler_to_event_reader.recv()
self.assertTrue(ph._handle_thread.is_alive())
self.assertEqual(msg, 1)
ph.stop()
self.assertFalse(ph._handle_thread.is_alive())
# Test handler handles given events
def testPapermillHandlerOngoingHandling(self)->None:
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
handler_to_event_us, handler_to_event_them = Pipe(duplex=True)
handler_to_job_us, handler_to_job_them = Pipe()
ph.to_runner_event = handler_to_event_them
ph.to_runner_job = handler_to_job_them
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = JupyterNotebookRecipe(
"recipe_one", COMPLETE_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, Rule)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
with self.assertRaises(AttributeError):
self.assertFalse(ph._handle_thread.is_alive())
ph.start()
if handler_to_event_us.poll(3):
msg = handler_to_event_us.recv()
self.assertEqual(msg, 1)
handler_to_event_us.send(event)
if handler_to_job_us.poll(3):
job_dir = handler_to_job_us.recv()
if handler_to_event_us.poll(3):
msg = handler_to_event_us.recv()
self.assertEqual(msg, 1)
ph.stop()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
class PythonTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -560,9 +644,9 @@ class PythonHandlerTests(unittest.TestCase):
# Test PythonHandler will handle given events
def testPythonHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -598,8 +682,8 @@ class PythonHandlerTests(unittest.TestCase):
ph.handle(event)
if from_handler_reader.poll(3):
job_dir = from_handler_reader.recv()
if from_handler_to_job_reader.poll(3):
job_dir = from_handler_to_job_reader.recv()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
@ -609,9 +693,9 @@ class PythonHandlerTests(unittest.TestCase):
# Test PythonHandler will create enough jobs from single sweep
def testPythonHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -652,8 +736,8 @@ class PythonHandlerTests(unittest.TestCase):
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
if from_handler_to_job_reader.poll(3):
jobs.append(from_handler_to_job_reader.recv())
else:
recieving = False
@ -674,9 +758,9 @@ class PythonHandlerTests(unittest.TestCase):
# Test PythonHandler will create enough jobs from multiple sweeps
def testPythonHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -722,8 +806,8 @@ class PythonHandlerTests(unittest.TestCase):
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
if from_handler_to_job_reader.poll(3):
jobs.append(from_handler_to_job_reader.recv())
else:
recieving = False
@ -840,7 +924,6 @@ class PythonHandlerTests(unittest.TestCase):
self.assertEqual(result, "124937.5")
# Test jobFunc doesn't execute with no args
def testJobFuncBadArgs(self)->None:
try:
@ -890,6 +973,90 @@ class PythonHandlerTests(unittest.TestCase):
})
self.assertTrue(status)
# Test handler starts and stops appropriatly
def testPythonHandlerStartStop(self)->None:
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
from_handler_to_event_reader, from_handler_to_event_writer = Pipe()
ph.to_runner_event = from_handler_to_event_writer
with self.assertRaises(AttributeError):
self.assertFalse(ph._handle_thread.is_alive())
ph.start()
if from_handler_to_event_reader.poll(3):
msg = from_handler_to_event_reader.recv()
self.assertTrue(ph._handle_thread.is_alive())
self.assertEqual(msg, 1)
ph.stop()
self.assertFalse(ph._handle_thread.is_alive())
# Test handler handles given events
def testPythonHandlerOngoingHandling(self)->None:
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
handler_to_event_us, handler_to_event_them = Pipe(duplex=True)
handler_to_job_us, handler_to_job_them = Pipe()
ph.to_runner_event = handler_to_event_them
ph.to_runner_job = handler_to_job_them
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, Rule)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
with self.assertRaises(AttributeError):
self.assertFalse(ph._handle_thread.is_alive())
ph.start()
if handler_to_event_us.poll(3):
msg = handler_to_event_us.recv()
self.assertEqual(msg, 1)
handler_to_event_us.send(event)
if handler_to_job_us.poll(3):
job_dir = handler_to_job_us.recv()
if handler_to_event_us.poll(3):
msg = handler_to_event_us.recv()
self.assertEqual(msg, 1)
ph.stop()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
class BashTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -973,9 +1140,9 @@ class BashHandlerTests(unittest.TestCase):
# Test BashHandler will handle given events
def testBashHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -1011,8 +1178,8 @@ class BashHandlerTests(unittest.TestCase):
ph.handle(event)
if from_handler_reader.poll(3):
job_dir = from_handler_reader.recv()
if from_handler_to_job_reader.poll(3):
job_dir = from_handler_to_job_reader.recv()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
@ -1022,9 +1189,9 @@ class BashHandlerTests(unittest.TestCase):
# Test BashHandler will create enough jobs from single sweep
def testBashHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -1065,8 +1232,8 @@ class BashHandlerTests(unittest.TestCase):
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
if from_handler_to_job_reader.poll(3):
jobs.append(from_handler_to_job_reader.recv())
else:
recieving = False
@ -1087,9 +1254,9 @@ class BashHandlerTests(unittest.TestCase):
# Test BashHandler will create enough jobs from multiple sweeps
def testBashHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
from_handler_to_job_reader, from_handler_to_job_writer = Pipe()
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
ph.to_runner_job = from_handler_to_job_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -1135,8 +1302,8 @@ class BashHandlerTests(unittest.TestCase):
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
if from_handler_to_job_reader.poll(3):
jobs.append(from_handler_to_job_reader.recv())
else:
recieving = False
@ -1299,3 +1466,86 @@ class BashHandlerTests(unittest.TestCase):
EVENT_RULE: rule
})
self.assertTrue(status)
# Test handler starts and stops appropriatly
def testBashHandlerStartStop(self)->None:
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
from_handler_to_event_reader, from_handler_to_event_writer = Pipe()
ph.to_runner_event = from_handler_to_event_writer
with self.assertRaises(AttributeError):
self.assertFalse(ph._handle_thread.is_alive())
ph.start()
if from_handler_to_event_reader.poll(3):
msg = from_handler_to_event_reader.recv()
self.assertTrue(ph._handle_thread.is_alive())
self.assertEqual(msg, 1)
ph.stop()
self.assertFalse(ph._handle_thread.is_alive())
# Test handler handles given events
def testBashHandlerOngoingHandling(self)->None:
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
handler_to_event_us, handler_to_event_them = Pipe(duplex=True)
handler_to_job_us, handler_to_job_them = Pipe()
ph.to_runner_event = handler_to_event_them
ph.to_runner_job = handler_to_job_them
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = BashRecipe(
"recipe_one", COMPLETE_BASH_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, Rule)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
with self.assertRaises(AttributeError):
self.assertFalse(ph._handle_thread.is_alive())
ph.start()
if handler_to_event_us.poll(3):
msg = handler_to_event_us.recv()
self.assertEqual(msg, 1)
handler_to_event_us.send(event)
if handler_to_job_us.poll(3):
job_dir = handler_to_job_us.recv()
if handler_to_event_us.poll(3):
msg = handler_to_event_us.recv()
self.assertEqual(msg, 1)
ph.stop()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)

File diff suppressed because it is too large Load Diff

View File

@ -10,7 +10,7 @@ from meow_base.core.meow import valid_event, valid_job, \
from meow_base.functionality.validation import check_type, \
check_implementation, valid_string, valid_dict, valid_list, \
valid_existing_file_path, valid_dir_path, valid_non_existing_path, \
check_callable
check_callable, valid_natural, valid_dict_multiple_types
from meow_base.core.vars import VALID_NAME_CHARS, SHA256, \
EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
@ -127,6 +127,36 @@ class ValidationTests(unittest.TestCase):
with self.assertRaises(ValueError):
valid_dict({"a": 0, "b": 1}, str, int, strict=True)
def testValidDictMultipleTypes(self)->None:
valid_dict_multiple_types(
{"a": 0, "b": 1},
str,
[int],
strict=False
)
valid_dict_multiple_types(
{"a": 0, "b": 1},
str,
[int, str],
strict=False
)
valid_dict_multiple_types(
{"a": 0, "b": 'a'},
str,
[int, str],
strict=False
)
with self.assertRaises(TypeError):
valid_dict_multiple_types(
{"a": 0, "b": 'a'},
str,
[int],
strict=False
)
# Test valid_list with sufficent lengths
def testValidListMinimum(self)->None:
valid_list([1, 2, 3], int)
@ -255,6 +285,18 @@ class ValidationTests(unittest.TestCase):
with self.assertRaises(TypeError):
check_callable("a")
# Test natural number check
def testValidNatural(self)->None:
valid_natural(0)
valid_natural(1)
with self.assertRaises(ValueError):
valid_natural(-1)
with self.assertRaises(TypeError):
valid_natural(1.0)
class MeowTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()