NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added naming to monitors, handlers and conductors so runners can identify them, in prep for in-workflow modification of patterns and recipes'

Browse Source
This commit is contained in:
PatchOfScotland
2023-03-16 13:53:01 +01:00
parent 9547df7612
commit f1f16ca3b8
12 changed files with 142 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
conductors/local_python_conductor.py
View File

@ -24,12 +24,12 @@ from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
class LocalPythonConductor(BaseConductor):
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR,
job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR)->None:
job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="")->None:
"""LocalPythonConductor Constructor. This should be used to execute
Python jobs, and will then pass any internal job runner files to the
output directory. Note that if this handler is given to a MeowRunner
object, the job_queue_dir and job_output_dir will be overwridden."""
super().__init__()
super().__init__(name=name)
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir
self._is_valid_job_output_dir(job_output_dir)

22
core/base_conductor.py
View File

@ -8,11 +8,17 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from meow_base.core.correctness.vars import get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation
from meow_base.core.correctness.vars import VALID_CONDUCTOR_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
valid_string
from meow_base.functionality.naming import generate_conductor_id
class BaseConductor:
# An identifier for a conductor within the runner. Can be manually set in
# the constructor, or autogenerated if no name provided.
name:str
# Directory where queued jobs are initially written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
@ -21,11 +27,15 @@ class BaseConductor:
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_output_dir:str
def __init__(self)->None:
def __init__(self, name:str="")->None:
"""BaseConductor Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).execute, BaseConductor)
check_implementation(type(self).valid_execute_criteria, BaseConductor)
if not name:
name = generate_conductor_id()
self._is_valid_name(name)
self.name = name
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
@ -35,6 +45,12 @@ class BaseConductor:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_CONDUCTOR_NAME_CHARS)
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
process it or not. Must be implemented by any child process."""

23
core/base_handler.py
View File

@ -8,11 +8,16 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation
from meow_base.core.correctness.vars import VALID_CHANNELS, \
VALID_HANDLER_NAME_CHARS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
valid_string
from meow_base.functionality.naming import generate_handler_id
class BaseHandler:
# An identifier for a handler within the runner. Can be manually set in
# the constructor, or autogenerated if no name provided.
name:str
# A channel for sending messages to the runner. Note that this will be
# overridden by a MeowRunner, if a handler instance is passed to it, and so
# does not need to be initialised within the handler itself.
@ -21,11 +26,15 @@ class BaseHandler:
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_queue_dir:str
def __init__(self)->None:
def __init__(self, name:str='')->None:
"""BaseHandler Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).handle, BaseHandler)
check_implementation(type(self).valid_handle_criteria, BaseHandler)
if not name:
name = generate_handler_id()
self._is_valid_name(name)
self.name = name
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
@ -35,6 +44,12 @@ class BaseHandler:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_HANDLER_NAME_CHARS)
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. Must be implemented by any child process."""

22
core/base_monitor.py
View File

@ -12,12 +12,18 @@ from typing import Union, Dict
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation
from meow_base.core.correctness.vars import VALID_CHANNELS, \
VALID_MONITOR_NAME_CHARS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
valid_string
from meow_base.functionality.meow import create_rules
from meow_base.functionality.naming import generate_monitor_id
class BaseMonitor:
# An identifier for a monitor within the runner. Can be manually set in
# the constructor, or autogenerated if no name provided.
name:str
# A collection of patterns
_patterns: Dict[str, BasePattern]
# A collection of recipes
@ -29,7 +35,7 @@ class BaseMonitor:
# monitor is passed to it.
to_runner: VALID_CHANNELS
def __init__(self, patterns:Dict[str,BasePattern],
recipes:Dict[str,BaseRecipe])->None:
recipes:Dict[str,BaseRecipe], name:str="")->None:
"""BaseMonitor Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
@ -53,6 +59,10 @@ class BaseMonitor:
self._patterns = deepcopy(patterns)
self._recipes = deepcopy(recipes)
self._rules = create_rules(patterns, recipes)
if not name:
name = generate_monitor_id()
self._is_valid_name(name)
self.name = name
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
@ -62,6 +72,12 @@ class BaseMonitor:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_MONITOR_NAME_CHARS)
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
"""Validation check for 'patterns' variable from main constructor. Must
be implemented by any child class."""

3
core/correctness/vars.py
View File

@ -21,6 +21,9 @@ CHAR_NUMERIC = '0123456789'
VALID_NAME_CHARS = CHAR_UPPERCASE + CHAR_LOWERCASE + CHAR_NUMERIC + "_-"
VALID_CONDUCTOR_NAME_CHARS = VALID_NAME_CHARS
VALID_HANDLER_NAME_CHARS = VALID_NAME_CHARS
VALID_MONITOR_NAME_CHARS = VALID_NAME_CHARS
VALID_RECIPE_NAME_CHARS = VALID_NAME_CHARS
VALID_PATTERN_NAME_CHARS = VALID_NAME_CHARS
VALID_RULE_NAME_CHARS = VALID_NAME_CHARS

9
functionality/naming.py
View File

@ -27,3 +27,12 @@ def generate_rule_id():
def generate_job_id():
return _generate_id(prefix="job_")
def generate_conductor_id():
return _generate_id(prefix="conductor_")
def generate_handler_id():
return _generate_id(prefix="handler_")
def generate_monitor_id():
return _generate_id(prefix="monitor_")

4
patterns/file_event_pattern.py
View File

@ -129,12 +129,12 @@ class WatchdogMonitor(BaseMonitor):
def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern],
recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1,
print:Any=sys.stdout, logging:int=0)->None:
name:str="", print:Any=sys.stdout, logging:int=0)->None:
"""WatchdogEventHandler Constructor. This uses the watchdog module to
monitor a directory and all its sub-directories. Watchdog will provide
the monitor with an caught events, with the monitor comparing them
against its rules, and informing the runner of match."""
super().__init__(patterns, recipes)
super().__init__(patterns, recipes, name=name)
self._is_valid_base_dir(base_dir)
self.base_dir = base_dir
check_type(settletime, int, hint="WatchdogMonitor.settletime")

4
recipes/jupyter_notebook_recipe.py
View File

@ -69,14 +69,14 @@ class PapermillHandler(BaseHandler):
debug_level:int
# Where print messages are sent
_print_target:Any
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR,
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
print:Any=sys.stdout, logging:int=0)->None:
"""PapermillHandler Constructor. This creats jobs to be executed using
the papermill module. This does not run as a continuous thread to
handle execution, but is invoked according to a factory pattern using
the handle function. Note that if this handler is given to a MeowRunner
object, the job_queue_dir will be overwridden."""
super().__init__()
super().__init__(name=name)
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir
self._print_target, self.debug_level = setup_debugging(print, logging)

4
recipes/python_recipe.py
View File

@ -59,14 +59,14 @@ class PythonHandler(BaseHandler):
debug_level:int
# Where print messages are sent
_print_target:Any
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR,
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
print:Any=sys.stdout, logging:int=0)->None:
"""PythonHandler Constructor. This creates jobs to be executed as
python functions. This does not run as a continuous thread to
handle execution, but is invoked according to a factory pattern using
the handle function. Note that if this handler is given to a MeowRunner
object, the job_queue_dir will be overwridden but its"""
super().__init__()
super().__init__(name=name)
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir
self._print_target, self.debug_level = setup_debugging(print, logging)

11
tests/test_conductors.py
View File

@ -38,10 +38,19 @@ class MeowTests(unittest.TestCase):
super().tearDown()
teardown()
# Test LocalPythonConductor creation and job types
# Test LocalPythonConductor creation
def testLocalPythonConductorCreation(self)->None:
LocalPythonConductor()
# Test LocalPythonConductor naming
def testLocalPythonConductorNaming(self)->None:
test_name = "test_name"
conductor = LocalPythonConductor(name=test_name)
self.assertEqual(conductor.name, test_name)
conductor = LocalPythonConductor()
self.assertTrue(conductor.name.startswith("conductor_"))
# Test LocalPythonConductor executes valid python jobs
def testLocalPythonConductorValidPythonJob(self)->None:
lpc = LocalPythonConductor(

20
tests/test_patterns.py
View File

@ -36,7 +36,7 @@ def recipes_equal(tester, recipe_one, recipe_two):
tester.assertEqual(recipe_one.source, recipe_two.source)
class CorrectnessTests(unittest.TestCase):
class FileEventPatternTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
@ -184,11 +184,29 @@ class CorrectnessTests(unittest.TestCase):
fep = FileEventPattern("name", "path", "recipe", "file",
sweep=bad_sweep)
class WatchdogMonitorTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test WatchdogMonitor created
def testWatchdogMonitorMinimum(self)->None:
from_monitor = Pipe()
WatchdogMonitor(TEST_MONITOR_BASE, {}, {}, from_monitor[1])
# Test WatchdogMonitor naming
def testWatchdogMonitorNaming(self)->None:
test_name = "test_name"
monitor = WatchdogMonitor(TEST_MONITOR_BASE, {}, {}, name=test_name)
self.assertEqual(monitor.name, test_name)
monitor = WatchdogMonitor(TEST_MONITOR_BASE, {}, {})
self.assertTrue(monitor.name.startswith("monitor_"))
# Test WatchdogMonitor identifies expected events in base directory
def testWatchdogMonitorEventIdentificaion(self)->None:
from_monitor_reader, from_monitor_writer = Pipe()

37
tests/test_recipes.py
View File

@ -111,10 +111,28 @@ class JupyterNotebookTests(unittest.TestCase):
"name", BAREBONES_NOTEBOOK, source=source)
self.assertEqual(jnr.source, source)
class PapermillHandlerTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test PapermillHandler can be created
def testPapermillHanderMinimum(self)->None:
PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
# Test PapermillHandler naming
def testPapermillHandlerNaming(self)->None:
test_name = "test_name"
handler = PapermillHandler(name=test_name)
self.assertEqual(handler.name, test_name)
handler = PapermillHandler()
self.assertTrue(handler.name.startswith("handler_"))
# Test PapermillHandler will handle given events
def testPapermillHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
@ -453,7 +471,6 @@ class JupyterNotebookTests(unittest.TestCase):
self.assertEqual(recipe.name, "name")
self.assertEqual(recipe.recipe, COMPLETE_NOTEBOOK)
class PythonTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -513,10 +530,28 @@ class PythonTests(unittest.TestCase):
"name", BAREBONES_PYTHON_SCRIPT, requirements=requirements)
self.assertEqual(pr.requirements, requirements)
class PythonHandlerTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test PythonHandler can be created
def testPythonHandlerMinimum(self)->None:
PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
# Test PythonHandler naming
def testPythonHandlerNaming(self)->None:
test_name = "test_name"
handler = PythonHandler(name=test_name)
self.assertEqual(handler.name, test_name)
handler = PythonHandler()
self.assertTrue(handler.name.startswith("handler_"))
# Test PythonHandler will handle given events
def testPythonHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()