diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index c470703..30305c1 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -24,12 +24,12 @@ from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \ class LocalPythonConductor(BaseConductor): def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, - job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR)->None: + job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="")->None: """LocalPythonConductor Constructor. This should be used to execute Python jobs, and will then pass any internal job runner files to the output directory. Note that if this handler is given to a MeowRunner object, the job_queue_dir and job_output_dir will be overwridden.""" - super().__init__() + super().__init__(name=name) self._is_valid_job_queue_dir(job_queue_dir) self.job_queue_dir = job_queue_dir self._is_valid_job_output_dir(job_output_dir) diff --git a/core/base_conductor.py b/core/base_conductor.py index 9745ce4..c590775 100644 --- a/core/base_conductor.py +++ b/core/base_conductor.py @@ -8,11 +8,17 @@ Author(s): David Marchant from typing import Any, Tuple, Dict -from meow_base.core.correctness.vars import get_drt_imp_msg -from meow_base.core.correctness.validation import check_implementation +from meow_base.core.correctness.vars import VALID_CONDUCTOR_NAME_CHARS, \ + get_drt_imp_msg +from meow_base.core.correctness.validation import check_implementation, \ + valid_string +from meow_base.functionality.naming import generate_conductor_id class BaseConductor: + # An identifier for a conductor within the runner. Can be manually set in + # the constructor, or autogenerated if no name provided. + name:str # Directory where queued jobs are initially written to. Note that this # will be overridden by a MeowRunner, if a handler instance is passed to # it, and so does not need to be initialised within the handler itself. @@ -21,11 +27,15 @@ class BaseConductor: # will be overridden by a MeowRunner, if a handler instance is passed to # it, and so does not need to be initialised within the handler itself. job_output_dir:str - def __init__(self)->None: + def __init__(self, name:str="")->None: """BaseConductor Constructor. This will check that any class inheriting from it implements its validation functions.""" check_implementation(type(self).execute, BaseConductor) check_implementation(type(self).valid_execute_criteria, BaseConductor) + if not name: + name = generate_conductor_id() + self._is_valid_name(name) + self.name = name def __new__(cls, *args, **kwargs): """A check that this base class is not instantiated itself, only @@ -35,6 +45,12 @@ class BaseConductor: raise TypeError(msg) return object.__new__(cls) + def _is_valid_name(self, name:str)->None: + """Validation check for 'name' variable from main constructor. Is + automatically called during initialisation. This does not need to be + overridden by child classes.""" + valid_string(name, VALID_CONDUCTOR_NAME_CHARS) + def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an job defintion, if this conductor can process it or not. Must be implemented by any child process.""" diff --git a/core/base_handler.py b/core/base_handler.py index b5e9db1..f1243d6 100644 --- a/core/base_handler.py +++ b/core/base_handler.py @@ -8,11 +8,16 @@ Author(s): David Marchant from typing import Any, Tuple, Dict -from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg -from meow_base.core.correctness.validation import check_implementation - +from meow_base.core.correctness.vars import VALID_CHANNELS, \ + VALID_HANDLER_NAME_CHARS, get_drt_imp_msg +from meow_base.core.correctness.validation import check_implementation, \ + valid_string +from meow_base.functionality.naming import generate_handler_id class BaseHandler: + # An identifier for a handler within the runner. Can be manually set in + # the constructor, or autogenerated if no name provided. + name:str # A channel for sending messages to the runner. Note that this will be # overridden by a MeowRunner, if a handler instance is passed to it, and so # does not need to be initialised within the handler itself. @@ -21,11 +26,15 @@ class BaseHandler: # will be overridden by a MeowRunner, if a handler instance is passed to # it, and so does not need to be initialised within the handler itself. job_queue_dir:str - def __init__(self)->None: + def __init__(self, name:str='')->None: """BaseHandler Constructor. This will check that any class inheriting from it implements its validation functions.""" check_implementation(type(self).handle, BaseHandler) check_implementation(type(self).valid_handle_criteria, BaseHandler) + if not name: + name = generate_handler_id() + self._is_valid_name(name) + self.name = name def __new__(cls, *args, **kwargs): """A check that this base class is not instantiated itself, only @@ -35,6 +44,12 @@ class BaseHandler: raise TypeError(msg) return object.__new__(cls) + def _is_valid_name(self, name:str)->None: + """Validation check for 'name' variable from main constructor. Is + automatically called during initialisation. This does not need to be + overridden by child classes.""" + valid_string(name, VALID_HANDLER_NAME_CHARS) + def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an event defintion, if this handler can process it or not. Must be implemented by any child process.""" diff --git a/core/base_monitor.py b/core/base_monitor.py index 4743413..7533694 100644 --- a/core/base_monitor.py +++ b/core/base_monitor.py @@ -12,12 +12,18 @@ from typing import Union, Dict from meow_base.core.base_pattern import BasePattern from meow_base.core.base_recipe import BaseRecipe from meow_base.core.base_rule import BaseRule -from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg -from meow_base.core.correctness.validation import check_implementation +from meow_base.core.correctness.vars import VALID_CHANNELS, \ + VALID_MONITOR_NAME_CHARS, get_drt_imp_msg +from meow_base.core.correctness.validation import check_implementation, \ + valid_string from meow_base.functionality.meow import create_rules +from meow_base.functionality.naming import generate_monitor_id class BaseMonitor: + # An identifier for a monitor within the runner. Can be manually set in + # the constructor, or autogenerated if no name provided. + name:str # A collection of patterns _patterns: Dict[str, BasePattern] # A collection of recipes @@ -29,7 +35,7 @@ class BaseMonitor: # monitor is passed to it. to_runner: VALID_CHANNELS def __init__(self, patterns:Dict[str,BasePattern], - recipes:Dict[str,BaseRecipe])->None: + recipes:Dict[str,BaseRecipe], name:str="")->None: """BaseMonitor Constructor. This will check that any class inheriting from it implements its validation functions. It will then call these on the input parameters.""" @@ -53,6 +59,10 @@ class BaseMonitor: self._patterns = deepcopy(patterns) self._recipes = deepcopy(recipes) self._rules = create_rules(patterns, recipes) + if not name: + name = generate_monitor_id() + self._is_valid_name(name) + self.name = name def __new__(cls, *args, **kwargs): """A check that this base class is not instantiated itself, only @@ -62,6 +72,12 @@ class BaseMonitor: raise TypeError(msg) return object.__new__(cls) + def _is_valid_name(self, name:str)->None: + """Validation check for 'name' variable from main constructor. Is + automatically called during initialisation. This does not need to be + overridden by child classes.""" + valid_string(name, VALID_MONITOR_NAME_CHARS) + def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None: """Validation check for 'patterns' variable from main constructor. Must be implemented by any child class.""" diff --git a/core/correctness/vars.py b/core/correctness/vars.py index d267cee..205ba13 100644 --- a/core/correctness/vars.py +++ b/core/correctness/vars.py @@ -21,6 +21,9 @@ CHAR_NUMERIC = '0123456789' VALID_NAME_CHARS = CHAR_UPPERCASE + CHAR_LOWERCASE + CHAR_NUMERIC + "_-" +VALID_CONDUCTOR_NAME_CHARS = VALID_NAME_CHARS +VALID_HANDLER_NAME_CHARS = VALID_NAME_CHARS +VALID_MONITOR_NAME_CHARS = VALID_NAME_CHARS VALID_RECIPE_NAME_CHARS = VALID_NAME_CHARS VALID_PATTERN_NAME_CHARS = VALID_NAME_CHARS VALID_RULE_NAME_CHARS = VALID_NAME_CHARS diff --git a/functionality/naming.py b/functionality/naming.py index 6f063f1..09c83c1 100644 --- a/functionality/naming.py +++ b/functionality/naming.py @@ -27,3 +27,12 @@ def generate_rule_id(): def generate_job_id(): return _generate_id(prefix="job_") + +def generate_conductor_id(): + return _generate_id(prefix="conductor_") + +def generate_handler_id(): + return _generate_id(prefix="handler_") + +def generate_monitor_id(): + return _generate_id(prefix="monitor_") diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index f069bf1..42756d8 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -129,12 +129,12 @@ class WatchdogMonitor(BaseMonitor): def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern], recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1, - print:Any=sys.stdout, logging:int=0)->None: + name:str="", print:Any=sys.stdout, logging:int=0)->None: """WatchdogEventHandler Constructor. This uses the watchdog module to monitor a directory and all its sub-directories. Watchdog will provide the monitor with an caught events, with the monitor comparing them against its rules, and informing the runner of match.""" - super().__init__(patterns, recipes) + super().__init__(patterns, recipes, name=name) self._is_valid_base_dir(base_dir) self.base_dir = base_dir check_type(settletime, int, hint="WatchdogMonitor.settletime") diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index 84b7a22..fbee432 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -69,14 +69,14 @@ class PapermillHandler(BaseHandler): debug_level:int # Where print messages are sent _print_target:Any - def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, + def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="", print:Any=sys.stdout, logging:int=0)->None: """PapermillHandler Constructor. This creats jobs to be executed using the papermill module. This does not run as a continuous thread to handle execution, but is invoked according to a factory pattern using the handle function. Note that if this handler is given to a MeowRunner object, the job_queue_dir will be overwridden.""" - super().__init__() + super().__init__(name=name) self._is_valid_job_queue_dir(job_queue_dir) self.job_queue_dir = job_queue_dir self._print_target, self.debug_level = setup_debugging(print, logging) diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index 07bf719..1630e32 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -59,14 +59,14 @@ class PythonHandler(BaseHandler): debug_level:int # Where print messages are sent _print_target:Any - def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, + def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="", print:Any=sys.stdout, logging:int=0)->None: """PythonHandler Constructor. This creates jobs to be executed as python functions. This does not run as a continuous thread to handle execution, but is invoked according to a factory pattern using the handle function. Note that if this handler is given to a MeowRunner object, the job_queue_dir will be overwridden but its""" - super().__init__() + super().__init__(name=name) self._is_valid_job_queue_dir(job_queue_dir) self.job_queue_dir = job_queue_dir self._print_target, self.debug_level = setup_debugging(print, logging) diff --git a/tests/test_conductors.py b/tests/test_conductors.py index 6e53354..d91233b 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -38,10 +38,19 @@ class MeowTests(unittest.TestCase): super().tearDown() teardown() - # Test LocalPythonConductor creation and job types + # Test LocalPythonConductor creation def testLocalPythonConductorCreation(self)->None: LocalPythonConductor() + # Test LocalPythonConductor naming + def testLocalPythonConductorNaming(self)->None: + test_name = "test_name" + conductor = LocalPythonConductor(name=test_name) + self.assertEqual(conductor.name, test_name) + + conductor = LocalPythonConductor() + self.assertTrue(conductor.name.startswith("conductor_")) + # Test LocalPythonConductor executes valid python jobs def testLocalPythonConductorValidPythonJob(self)->None: lpc = LocalPythonConductor( diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 6359d1c..a79677f 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -36,7 +36,7 @@ def recipes_equal(tester, recipe_one, recipe_two): tester.assertEqual(recipe_one.source, recipe_two.source) -class CorrectnessTests(unittest.TestCase): +class FileEventPatternTests(unittest.TestCase): def setUp(self)->None: super().setUp() setup() @@ -184,11 +184,29 @@ class CorrectnessTests(unittest.TestCase): fep = FileEventPattern("name", "path", "recipe", "file", sweep=bad_sweep) +class WatchdogMonitorTests(unittest.TestCase): + def setUp(self)->None: + super().setUp() + setup() + + def tearDown(self)->None: + super().tearDown() + teardown() + # Test WatchdogMonitor created def testWatchdogMonitorMinimum(self)->None: from_monitor = Pipe() WatchdogMonitor(TEST_MONITOR_BASE, {}, {}, from_monitor[1]) + # Test WatchdogMonitor naming + def testWatchdogMonitorNaming(self)->None: + test_name = "test_name" + monitor = WatchdogMonitor(TEST_MONITOR_BASE, {}, {}, name=test_name) + self.assertEqual(monitor.name, test_name) + + monitor = WatchdogMonitor(TEST_MONITOR_BASE, {}, {}) + self.assertTrue(monitor.name.startswith("monitor_")) + # Test WatchdogMonitor identifies expected events in base directory def testWatchdogMonitorEventIdentificaion(self)->None: from_monitor_reader, from_monitor_writer = Pipe() diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 044ec60..2b91ee2 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -111,10 +111,28 @@ class JupyterNotebookTests(unittest.TestCase): "name", BAREBONES_NOTEBOOK, source=source) self.assertEqual(jnr.source, source) +class PapermillHandlerTests(unittest.TestCase): + def setUp(self)->None: + super().setUp() + setup() + + def tearDown(self)->None: + super().tearDown() + teardown() + # Test PapermillHandler can be created def testPapermillHanderMinimum(self)->None: PapermillHandler(job_queue_dir=TEST_JOB_QUEUE) + # Test PapermillHandler naming + def testPapermillHandlerNaming(self)->None: + test_name = "test_name" + handler = PapermillHandler(name=test_name) + self.assertEqual(handler.name, test_name) + + handler = PapermillHandler() + self.assertTrue(handler.name.startswith("handler_")) + # Test PapermillHandler will handle given events def testPapermillHandlerHandling(self)->None: from_handler_reader, from_handler_writer = Pipe() @@ -453,7 +471,6 @@ class JupyterNotebookTests(unittest.TestCase): self.assertEqual(recipe.name, "name") self.assertEqual(recipe.recipe, COMPLETE_NOTEBOOK) - class PythonTests(unittest.TestCase): def setUp(self)->None: super().setUp() @@ -513,10 +530,28 @@ class PythonTests(unittest.TestCase): "name", BAREBONES_PYTHON_SCRIPT, requirements=requirements) self.assertEqual(pr.requirements, requirements) +class PythonHandlerTests(unittest.TestCase): + def setUp(self)->None: + super().setUp() + setup() + + def tearDown(self)->None: + super().tearDown() + teardown() + # Test PythonHandler can be created def testPythonHandlerMinimum(self)->None: PythonHandler(job_queue_dir=TEST_JOB_QUEUE) + # Test PythonHandler naming + def testPythonHandlerNaming(self)->None: + test_name = "test_name" + handler = PythonHandler(name=test_name) + self.assertEqual(handler.name, test_name) + + handler = PythonHandler() + self.assertTrue(handler.name.startswith("handler_")) + # Test PythonHandler will handle given events def testPythonHandlerHandling(self)->None: from_handler_reader, from_handler_writer = Pipe()