added naming to monitors, handlers and conductors so runners can identify them, in prep for in-workflow modification of patterns and recipes'

conductors/local_python_conductor.py

@@ -24,12 +24,12 @@ from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
 
 class LocalPythonConductor(BaseConductor):
     def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
-            job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR)->None:
+            job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="")->None:
         """LocalPythonConductor Constructor. This should be used to execute 
         Python jobs, and will then pass any internal job runner files to the 
         output directory. Note that if this handler is given to a MeowRunner
         object, the job_queue_dir and job_output_dir will be overwridden."""
-        super().__init__()
+        super().__init__(name=name)
         self._is_valid_job_queue_dir(job_queue_dir)
         self.job_queue_dir = job_queue_dir
         self._is_valid_job_output_dir(job_output_dir)

core/base_conductor.py

@@ -8,11 +8,17 @@ Author(s): David Marchant
 
 from typing import Any, Tuple, Dict
 
-from meow_base.core.correctness.vars import get_drt_imp_msg
+from meow_base.core.correctness.vars import VALID_CONDUCTOR_NAME_CHARS, \
-from meow_base.core.correctness.validation import check_implementation
+    get_drt_imp_msg
+from meow_base.core.correctness.validation import check_implementation, \
+    valid_string
+from meow_base.functionality.naming import generate_conductor_id
 
 
 class BaseConductor:
+    # An identifier for a conductor within the runner. Can be manually set in 
+    # the constructor, or autogenerated if no name provided.
+    name:str
     # Directory where queued jobs are initially written to. Note that this 
     # will be overridden by a MeowRunner, if a handler instance is passed to 
     # it, and so does not need to be initialised within the handler itself.
@@ -21,11 +27,15 @@ class BaseConductor:
     # will be overridden by a MeowRunner, if a handler instance is passed to 
     # it, and so does not need to be initialised within the handler itself.
     job_output_dir:str
-    def __init__(self)->None:
+    def __init__(self, name:str="")->None:
         """BaseConductor Constructor. This will check that any class inheriting
         from it implements its validation functions."""
         check_implementation(type(self).execute, BaseConductor)
         check_implementation(type(self).valid_execute_criteria, BaseConductor)
+        if not name:
+            name = generate_conductor_id()
+        self._is_valid_name(name)
+        self.name = name    
 
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 
@@ -35,6 +45,12 @@ class BaseConductor:
             raise TypeError(msg)
         return object.__new__(cls)
 
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_CONDUCTOR_NAME_CHARS)
+
     def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
         """Function to determine given an job defintion, if this conductor can 
         process it or not. Must be implemented by any child process."""

core/base_handler.py

@@ -8,11 +8,16 @@ Author(s): David Marchant
 
 from typing import Any, Tuple, Dict
 
-from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg 
+from meow_base.core.correctness.vars import VALID_CHANNELS, \
-from meow_base.core.correctness.validation import check_implementation
+    VALID_HANDLER_NAME_CHARS, get_drt_imp_msg
-
+from meow_base.core.correctness.validation import check_implementation, \
+    valid_string
+from meow_base.functionality.naming import generate_handler_id
 
 class BaseHandler:
+    # An identifier for a handler within the runner. Can be manually set in 
+    # the constructor, or autogenerated if no name provided.
+    name:str
     # A channel for sending messages to the runner. Note that this will be 
     # overridden by a MeowRunner, if a handler instance is passed to it, and so
     # does not need to be initialised within the handler itself.
@@ -21,11 +26,15 @@ class BaseHandler:
     # will be overridden by a MeowRunner, if a handler instance is passed to 
     # it, and so does not need to be initialised within the handler itself.
     job_queue_dir:str
-    def __init__(self)->None:
+    def __init__(self, name:str='')->None:
         """BaseHandler Constructor. This will check that any class inheriting 
         from it implements its validation functions."""
         check_implementation(type(self).handle, BaseHandler)
         check_implementation(type(self).valid_handle_criteria, BaseHandler)
+        if not name:
+            name = generate_handler_id()
+        self._is_valid_name(name)
+        self.name = name    
 
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 
@@ -35,6 +44,12 @@ class BaseHandler:
             raise TypeError(msg)
         return object.__new__(cls)
 
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_HANDLER_NAME_CHARS)
+
     def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
         """Function to determine given an event defintion, if this handler can 
         process it or not. Must be implemented by any child process."""

core/base_monitor.py

@@ -12,12 +12,18 @@ from typing import Union, Dict
 from meow_base.core.base_pattern import BasePattern
 from meow_base.core.base_recipe import BaseRecipe
 from meow_base.core.base_rule import BaseRule
-from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg 
+from meow_base.core.correctness.vars import VALID_CHANNELS, \
-from meow_base.core.correctness.validation import check_implementation
+    VALID_MONITOR_NAME_CHARS, get_drt_imp_msg 
+from meow_base.core.correctness.validation import check_implementation, \
+    valid_string
 from meow_base.functionality.meow import create_rules
+from meow_base.functionality.naming import generate_monitor_id
 
 
 class BaseMonitor:
+    # An identifier for a monitor within the runner. Can be manually set in 
+    # the constructor, or autogenerated if no name provided.
+    name:str
     # A collection of patterns
     _patterns: Dict[str, BasePattern]
     # A collection of recipes
@@ -29,7 +35,7 @@ class BaseMonitor:
     # monitor is passed to it.
     to_runner: VALID_CHANNELS
     def __init__(self, patterns:Dict[str,BasePattern], 
-            recipes:Dict[str,BaseRecipe])->None:
+            recipes:Dict[str,BaseRecipe], name:str="")->None:
         """BaseMonitor Constructor. This will check that any class inheriting 
         from it implements its validation functions. It will then call these on
         the input parameters."""
@@ -53,6 +59,10 @@ class BaseMonitor:
         self._patterns = deepcopy(patterns)
         self._recipes = deepcopy(recipes)
         self._rules = create_rules(patterns, recipes)
+        if not name:
+            name = generate_monitor_id()
+        self._is_valid_name(name)
+        self.name = name    
         
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 
@@ -62,6 +72,12 @@ class BaseMonitor:
             raise TypeError(msg)
         return object.__new__(cls)
 
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_MONITOR_NAME_CHARS)
+
     def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
         """Validation check for 'patterns' variable from main constructor. Must
         be implemented by any child class."""

core/correctness/vars.py

@@ -21,6 +21,9 @@ CHAR_NUMERIC = '0123456789'
 
 VALID_NAME_CHARS = CHAR_UPPERCASE + CHAR_LOWERCASE + CHAR_NUMERIC + "_-"
 
+VALID_CONDUCTOR_NAME_CHARS = VALID_NAME_CHARS
+VALID_HANDLER_NAME_CHARS = VALID_NAME_CHARS
+VALID_MONITOR_NAME_CHARS = VALID_NAME_CHARS
 VALID_RECIPE_NAME_CHARS = VALID_NAME_CHARS
 VALID_PATTERN_NAME_CHARS = VALID_NAME_CHARS
 VALID_RULE_NAME_CHARS = VALID_NAME_CHARS

functionality/naming.py

@@ -27,3 +27,12 @@ def generate_rule_id():
 
 def generate_job_id():
     return _generate_id(prefix="job_")
+
+def generate_conductor_id():
+    return _generate_id(prefix="conductor_")
+
+def generate_handler_id():
+    return _generate_id(prefix="handler_")
+
+def generate_monitor_id():
+    return _generate_id(prefix="monitor_")

patterns/file_event_pattern.py

@@ -129,12 +129,12 @@ class WatchdogMonitor(BaseMonitor):
 
     def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern], 
             recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1, 
-            print:Any=sys.stdout, logging:int=0)->None:
+            name:str="", print:Any=sys.stdout, logging:int=0)->None:
         """WatchdogEventHandler Constructor. This uses the watchdog module to 
         monitor a directory and all its sub-directories. Watchdog will provide 
         the monitor with an caught events, with the monitor comparing them 
         against its rules, and informing the runner of match."""
-        super().__init__(patterns, recipes)
+        super().__init__(patterns, recipes, name=name)
         self._is_valid_base_dir(base_dir)
         self.base_dir = base_dir
         check_type(settletime, int, hint="WatchdogMonitor.settletime")

recipes/jupyter_notebook_recipe.py

@@ -69,14 +69,14 @@ class PapermillHandler(BaseHandler):
     debug_level:int
     # Where print messages are sent
     _print_target:Any
-    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
             print:Any=sys.stdout, logging:int=0)->None:
         """PapermillHandler Constructor. This creats jobs to be executed using 
         the papermill module. This does not run as a continuous thread to 
         handle execution, but is invoked according to a factory pattern using 
         the handle function. Note that if this handler is given to a MeowRunner
         object, the job_queue_dir will be overwridden."""
-        super().__init__()
+        super().__init__(name=name)
         self._is_valid_job_queue_dir(job_queue_dir)
         self.job_queue_dir = job_queue_dir
         self._print_target, self.debug_level = setup_debugging(print, logging)

recipes/python_recipe.py

@@ -59,14 +59,14 @@ class PythonHandler(BaseHandler):
     debug_level:int
     # Where print messages are sent
     _print_target:Any
-    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
             print:Any=sys.stdout, logging:int=0)->None:
         """PythonHandler Constructor. This creates jobs to be executed as 
         python functions. This does not run as a continuous thread to 
         handle execution, but is invoked according to a factory pattern using 
         the handle function. Note that if this handler is given to a MeowRunner
         object, the job_queue_dir will be overwridden but its"""
-        super().__init__()
+        super().__init__(name=name)
         self._is_valid_job_queue_dir(job_queue_dir)
         self.job_queue_dir = job_queue_dir
         self._print_target, self.debug_level = setup_debugging(print, logging)

tests/test_conductors.py

@@ -38,10 +38,19 @@ class MeowTests(unittest.TestCase):
         super().tearDown()
         teardown()
         
-    # Test LocalPythonConductor creation and job types
+    # Test LocalPythonConductor creation
     def testLocalPythonConductorCreation(self)->None:
         LocalPythonConductor()
 
+    # Test LocalPythonConductor naming
+    def testLocalPythonConductorNaming(self)->None:
+        test_name = "test_name"
+        conductor = LocalPythonConductor(name=test_name)
+        self.assertEqual(conductor.name, test_name)
+
+        conductor = LocalPythonConductor()
+        self.assertTrue(conductor.name.startswith("conductor_"))
+
     # Test LocalPythonConductor executes valid python jobs
     def testLocalPythonConductorValidPythonJob(self)->None:
         lpc = LocalPythonConductor(

tests/test_patterns.py

@@ -36,7 +36,7 @@ def recipes_equal(tester, recipe_one, recipe_two):
     tester.assertEqual(recipe_one.source, recipe_two.source)
 
 
-class CorrectnessTests(unittest.TestCase):
+class FileEventPatternTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
         setup()
@@ -184,11 +184,29 @@ class CorrectnessTests(unittest.TestCase):
             fep = FileEventPattern("name", "path", "recipe", "file", 
                 sweep=bad_sweep)
 
+class WatchdogMonitorTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+
     # Test WatchdogMonitor created 
     def testWatchdogMonitorMinimum(self)->None:
         from_monitor = Pipe()
         WatchdogMonitor(TEST_MONITOR_BASE, {}, {}, from_monitor[1])
 
+    # Test WatchdogMonitor naming
+    def testWatchdogMonitorNaming(self)->None:
+        test_name = "test_name"
+        monitor = WatchdogMonitor(TEST_MONITOR_BASE, {}, {}, name=test_name)
+        self.assertEqual(monitor.name, test_name)
+
+        monitor = WatchdogMonitor(TEST_MONITOR_BASE, {}, {})
+        self.assertTrue(monitor.name.startswith("monitor_"))
+
     # Test WatchdogMonitor identifies expected events in base directory
     def testWatchdogMonitorEventIdentificaion(self)->None:
         from_monitor_reader, from_monitor_writer = Pipe()

tests/test_recipes.py

@@ -111,10 +111,28 @@ class JupyterNotebookTests(unittest.TestCase):
             "name", BAREBONES_NOTEBOOK, source=source)
         self.assertEqual(jnr.source, source)
 
+class PapermillHandlerTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+
     # Test PapermillHandler can be created
     def testPapermillHanderMinimum(self)->None:
         PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
 
+    # Test PapermillHandler naming
+    def testPapermillHandlerNaming(self)->None:
+        test_name = "test_name"
+        handler = PapermillHandler(name=test_name)
+        self.assertEqual(handler.name, test_name)
+
+        handler = PapermillHandler()
+        self.assertTrue(handler.name.startswith("handler_"))
+
     # Test PapermillHandler will handle given events
     def testPapermillHandlerHandling(self)->None:
         from_handler_reader, from_handler_writer = Pipe()
@@ -453,7 +471,6 @@ class JupyterNotebookTests(unittest.TestCase):
         self.assertEqual(recipe.name, "name")
         self.assertEqual(recipe.recipe, COMPLETE_NOTEBOOK)
 
-
 class PythonTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
@@ -513,10 +530,28 @@ class PythonTests(unittest.TestCase):
             "name", BAREBONES_PYTHON_SCRIPT, requirements=requirements)
         self.assertEqual(pr.requirements, requirements)
 
+class PythonHandlerTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+
     # Test PythonHandler can be created
     def testPythonHandlerMinimum(self)->None:
         PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
 
+    # Test PythonHandler naming
+    def testPythonHandlerNaming(self)->None:
+        test_name = "test_name"
+        handler = PythonHandler(name=test_name)
+        self.assertEqual(handler.name, test_name)
+
+        handler = PythonHandler()
+        self.assertTrue(handler.name.startswith("handler_"))
+
     # Test PythonHandler will handle given events
     def testPythonHandlerHandling(self)->None:
         from_handler_reader, from_handler_writer = Pipe()