also refactored core.meow into seperate files in hope that it'll help solve circular imports

.gitignore

@@ -51,7 +51,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 tests/test_monitor_base
-tests/test_job_queue_dir_dir_dir_dir_dir_dir_dir_dir_dir_dir
+tests/test_job_queue_dir
 tests/test_job_output
 
 # Translations

conductors/local_python_conductor.py

@@ -11,12 +11,12 @@ import shutil
 from datetime import datetime
 from typing import Any, Tuple, Dict
 
+from core.base_conductor import BaseConductor
 from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
     STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
     STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
     JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
 from core.correctness.validation import valid_job, valid_dir_path
-from core.meow import BaseConductor
 from functionality.file_io import make_dir, read_yaml, write_file, write_yaml
 
 class LocalPythonConductor(BaseConductor):

core/base_conductor.py

@@ -0,0 +1,46 @@
+
+"""
+This file contains the base MEOW conductor defintion. This should be inherited 
+from for all conductor instances.
+
+Author(s): David Marchant
+"""
+
+from typing import Any, Tuple, Dict
+
+from core.correctness.vars import get_drt_imp_msg
+from core.correctness.validation import check_implementation
+
+
+class BaseConductor:
+    # Directory where queued jobs are initially written to. Note that this 
+    # will be overridden by a MeowRunner, if a handler instance is passed to 
+    # it, and so does not need to be initialised within the handler itself.
+    job_queue_dir:str
+    # Directory where completed jobs are finally written to. Note that this 
+    # will be overridden by a MeowRunner, if a handler instance is passed to 
+    # it, and so does not need to be initialised within the handler itself.
+    job_output_dir:str
+    def __init__(self)->None:
+        """BaseConductor Constructor. This will check that any class inheriting
+        from it implements its validation functions."""
+        check_implementation(type(self).execute, BaseConductor)
+        check_implementation(type(self).valid_execute_criteria, BaseConductor)
+
+    def __new__(cls, *args, **kwargs):
+        """A check that this base class is not instantiated itself, only 
+        inherited from"""
+        if cls is BaseConductor:
+            msg = get_drt_imp_msg(BaseConductor)
+            raise TypeError(msg)
+        return object.__new__(cls)
+
+    def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
+        """Function to determine given an job defintion, if this conductor can 
+        process it or not. Must be implemented by any child process."""
+        pass
+
+    def execute(self, job_dir:str)->None:
+        """Function to execute a given job directory. Must be implemented by 
+        any child process."""
+        pass

core/base_handler.py

@@ -0,0 +1,46 @@
+
+"""
+This file contains the base MEOW handler defintion. This should be inherited 
+from for all handler instances.
+
+Author(s): David Marchant
+"""
+
+from typing import Any, Tuple, Dict
+
+from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
+from core.correctness.validation import check_implementation
+
+
+class BaseHandler:
+    # A channel for sending messages to the runner. Note that this will be 
+    # overridden by a MeowRunner, if a handler instance is passed to it, and so
+    # does not need to be initialised within the handler itself.
+    to_runner: VALID_CHANNELS
+    # Directory where queued jobs are initially written to. Note that this 
+    # will be overridden by a MeowRunner, if a handler instance is passed to 
+    # it, and so does not need to be initialised within the handler itself.
+    job_queue_dir:str
+    def __init__(self)->None:
+        """BaseHandler Constructor. This will check that any class inheriting 
+        from it implements its validation functions."""
+        check_implementation(type(self).handle, BaseHandler)
+        check_implementation(type(self).valid_handle_criteria, BaseHandler)
+
+    def __new__(cls, *args, **kwargs):
+        """A check that this base class is not instantiated itself, only 
+        inherited from"""
+        if cls is BaseHandler:
+            msg = get_drt_imp_msg(BaseHandler)
+            raise TypeError(msg)
+        return object.__new__(cls)
+
+    def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
+        """Function to determine given an event defintion, if this handler can 
+        process it or not. Must be implemented by any child process."""
+        pass
+
+    def handle(self, event:Dict[str,Any])->None:
+        """Function to handle a given event. Must be implemented by any child 
+        process."""
+        pass

core/base_monitor.py

@@ -0,0 +1,128 @@
+
+"""
+This file contains the base MEOW monitor defintion. This should be inherited 
+from for all monitor instances.
+
+Author(s): David Marchant
+"""
+
+from copy import deepcopy
+from typing import Union, Dict
+
+from core.base_pattern import BasePattern
+from core.base_recipe import BaseRecipe
+from core.base_rule import BaseRule
+from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
+from core.correctness.validation import check_implementation
+from functionality.meow import create_rules
+
+
+class BaseMonitor:
+    # A collection of patterns
+    _patterns: Dict[str, BasePattern]
+    # A collection of recipes
+    _recipes: Dict[str, BaseRecipe]
+    # A collection of rules derived from _patterns and _recipes
+    _rules: Dict[str, BaseRule]
+    # A channel for sending messages to the runner. Note that this is not 
+    # initialised within the constructor, but within the runner when passed the
+    # monitor is passed to it.
+    to_runner: VALID_CHANNELS
+    def __init__(self, patterns:Dict[str,BasePattern], 
+            recipes:Dict[str,BaseRecipe])->None:
+        """BaseMonitor Constructor. This will check that any class inheriting 
+        from it implements its validation functions. It will then call these on
+        the input parameters."""
+        check_implementation(type(self).start, BaseMonitor)
+        check_implementation(type(self).stop, BaseMonitor)
+        check_implementation(type(self)._is_valid_patterns, BaseMonitor)
+        self._is_valid_patterns(patterns)
+        check_implementation(type(self)._is_valid_recipes, BaseMonitor)
+        self._is_valid_recipes(recipes)
+        check_implementation(type(self).add_pattern, BaseMonitor)
+        check_implementation(type(self).update_pattern, BaseMonitor)
+        check_implementation(type(self).remove_pattern, BaseMonitor)
+        check_implementation(type(self).get_patterns, BaseMonitor)
+        check_implementation(type(self).add_recipe, BaseMonitor)
+        check_implementation(type(self).update_recipe, BaseMonitor)
+        check_implementation(type(self).remove_recipe, BaseMonitor)
+        check_implementation(type(self).get_recipes, BaseMonitor)
+        check_implementation(type(self).get_rules, BaseMonitor)
+        # Ensure that patterns and recipes cannot be trivially modified from 
+        # outside the monitor, as this will cause internal consistency issues
+        self._patterns = deepcopy(patterns)
+        self._recipes = deepcopy(recipes)
+        self._rules = create_rules(patterns, recipes)
+        
+    def __new__(cls, *args, **kwargs):
+        """A check that this base class is not instantiated itself, only 
+        inherited from"""
+        if cls is BaseMonitor:
+            msg = get_drt_imp_msg(BaseMonitor)
+            raise TypeError(msg)
+        return object.__new__(cls)
+
+    def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
+        """Validation check for 'patterns' variable from main constructor. Must
+        be implemented by any child class."""
+        pass
+
+    def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
+        """Validation check for 'recipes' variable from main constructor. Must 
+        be implemented by any child class."""
+        pass
+
+    def start(self)->None:
+        """Function to start the monitor as an ongoing process/thread. Must be 
+        implemented by any child process"""
+        pass
+
+    def stop(self)->None:
+        """Function to stop the monitor as an ongoing process/thread. Must be 
+        implemented by any child process"""
+        pass
+
+    def add_pattern(self, pattern:BasePattern)->None:
+        """Function to add a pattern to the current definitions. Must be 
+        implemented by any child process."""
+        pass
+
+    def update_pattern(self, pattern:BasePattern)->None:
+        """Function to update a pattern in the current definitions. Must be 
+        implemented by any child process."""
+        pass
+
+    def remove_pattern(self, pattern:Union[str,BasePattern])->None:
+        """Function to remove a pattern from the current definitions. Must be 
+        implemented by any child process."""
+        pass
+
+    def get_patterns(self)->Dict[str,BasePattern]:
+        """Function to get a dictionary of all current pattern definitions. 
+        Must be implemented by any child process."""
+        pass
+
+    def add_recipe(self, recipe:BaseRecipe)->None:
+        """Function to add a recipe to the current definitions. Must be 
+        implemented by any child process."""
+        pass
+
+    def update_recipe(self, recipe:BaseRecipe)->None:
+        """Function to update a recipe in the current definitions. Must be 
+        implemented by any child process."""
+        pass
+
+    def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
+        """Function to remove a recipe from the current definitions. Must be 
+        implemented by any child process."""
+        pass
+
+    def get_recipes(self)->Dict[str,BaseRecipe]:
+        """Function to get a dictionary of all current recipe definitions. 
+        Must be implemented by any child process."""
+        pass
+
+    def get_rules(self)->Dict[str,BaseRule]:
+        """Function to get a dictionary of all current rule definitions. 
+        Must be implemented by any child process."""
+        pass

core/base_pattern.py

@@ -0,0 +1,141 @@
+
+"""
+This file contains the base MEOW pattern defintion. This should be inherited 
+from for all pattern instances.
+
+Author(s): David Marchant
+"""
+
+import itertools
+
+from typing import Any, Union, Tuple, Dict, List
+
+from core.correctness.vars import get_drt_imp_msg, \
+    VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
+from core.correctness.validation import valid_string, check_type, \
+    check_implementation, valid_dict
+
+
+class BasePattern:
+    # A unique identifier for the pattern
+    name:str
+    # An identifier of a recipe
+    recipe:str
+    # Parameters to be overridden in the recipe
+    parameters:Dict[str,Any]
+    # Parameters showing the potential outputs of a recipe
+    outputs:Dict[str,Any]
+    # A collection of variables to be swept over for job scheduling
+    sweep:Dict[str,Any]
+
+    def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={}, 
+            outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}):
+        """BasePattern Constructor. This will check that any class inheriting 
+        from it implements its validation functions. It will then call these on
+        the input parameters."""
+        check_implementation(type(self)._is_valid_recipe, BasePattern)
+        check_implementation(type(self)._is_valid_parameters, BasePattern)
+        check_implementation(type(self)._is_valid_output, BasePattern)
+        self._is_valid_name(name)
+        self.name = name
+        self._is_valid_recipe(recipe)
+        self.recipe = recipe
+        self._is_valid_parameters(parameters)
+        self.parameters = parameters
+        self._is_valid_output(outputs)
+        self.outputs = outputs
+        self._is_valid_sweep(sweep)
+        self.sweep = sweep
+
+    def __new__(cls, *args, **kwargs):
+        """A check that this base class is not instantiated itself, only 
+        inherited from"""
+        if cls is BasePattern:
+            msg = get_drt_imp_msg(BasePattern)
+            raise TypeError(msg)
+        return object.__new__(cls)
+
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_PATTERN_NAME_CHARS)
+
+    def _is_valid_recipe(self, recipe:Any)->None:
+        """Validation check for 'recipe' variable from main constructor. Must 
+        be implemented by any child class."""
+        pass
+
+    def _is_valid_parameters(self, parameters:Any)->None:
+        """Validation check for 'parameters' variable from main constructor. 
+        Must be implemented by any child class."""
+        pass
+
+    def _is_valid_output(self, outputs:Any)->None:
+        """Validation check for 'outputs' variable from main constructor. Must 
+        be implemented by any child class."""
+        pass
+
+    def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None:
+        """Validation check for 'sweep' variable from main constructor. This 
+        function is implemented to check for the types given in the signature, 
+        and must be overridden if these differ."""
+        check_type(sweep, Dict, hint="BasePattern.sweep")
+        if not sweep:
+            return
+        for _, v in sweep.items():
+            valid_dict(
+                v, str, Any, [
+                    SWEEP_START, SWEEP_STOP, SWEEP_JUMP
+                ], strict=True)
+
+            check_type(
+                v[SWEEP_START], 
+                expected_type=int, 
+                alt_types=[float, complex],
+                hint=f"BasePattern.sweep[{SWEEP_START}]"
+            )
+            check_type(
+                v[SWEEP_STOP], 
+                expected_type=int, 
+                alt_types=[float, complex],
+                hint=f"BasePattern.sweep[{SWEEP_STOP}]"
+            )
+            check_type(
+                v[SWEEP_JUMP], 
+                expected_type=int, 
+                alt_types=[float, complex],
+                hint=f"BasePattern.sweep[{SWEEP_JUMP}]"
+            )
+            # Try to check that this loop is not infinite
+            if v[SWEEP_JUMP] == 0:
+                raise ValueError(
+                    f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
+                )
+            elif v[SWEEP_JUMP] > 0:
+                if not v[SWEEP_STOP] > v[SWEEP_START]:
+                    raise ValueError(
+                        f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
+                        "value where the end point is smaller than the start."
+                    )
+            elif v[SWEEP_JUMP] < 0:
+                if not v[SWEEP_STOP] < v[SWEEP_START]:
+                    raise ValueError(
+                        f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
+                        "value where the end point is smaller than the start."
+                    )
+
+    def expand_sweeps(self)->List[Tuple[str,Any]]:
+        """Function to get all combinations of sweep parameters"""
+        values_dict = {}
+        # get a collection of a individual sweep values
+        for var, val in self.sweep.items():
+            values_dict[var] = []
+            par_val = val[SWEEP_START]
+            while par_val <= val[SWEEP_STOP]:
+                values_dict[var].append((var, par_val))
+                par_val += val[SWEEP_JUMP]
+
+        # combine all combinations of sweep values
+        return list(itertools.product(
+            *[v for v in values_dict.values()]))

core/base_recipe.py

@@ -0,0 +1,68 @@
+
+"""
+This file contains the base MEOW recipe defintion. This should be inherited 
+from for all recipe instances.
+
+Author(s): David Marchant
+"""
+
+from typing import Any, Dict
+
+from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
+from core.correctness.validation import valid_string, check_implementation
+
+
+class BaseRecipe:
+    # A unique identifier for the recipe
+    name:str
+    # Actual code to run
+    recipe:Any
+    # Possible parameters that could be overridden by a Pattern 
+    parameters:Dict[str, Any]
+    # Additional configuration options
+    requirements:Dict[str, Any]
+    def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, 
+            requirements:Dict[str,Any]={}):
+        """BaseRecipe Constructor. This will check that any class inheriting 
+        from it implements its validation functions. It will then call these on
+        the input parameters."""
+        check_implementation(type(self)._is_valid_recipe, BaseRecipe)
+        check_implementation(type(self)._is_valid_parameters, BaseRecipe)
+        check_implementation(type(self)._is_valid_requirements, BaseRecipe)
+        self._is_valid_name(name)
+        self.name = name
+        self._is_valid_recipe(recipe)
+        self.recipe = recipe
+        self._is_valid_parameters(parameters)
+        self.parameters = parameters
+        self._is_valid_requirements(requirements)
+        self.requirements = requirements
+
+    def __new__(cls, *args, **kwargs):
+        """A check that this base class is not instantiated itself, only 
+        inherited from"""
+        if cls is BaseRecipe:
+            msg = get_drt_imp_msg(BaseRecipe)
+            raise TypeError(msg)
+        return object.__new__(cls)
+
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_RECIPE_NAME_CHARS)
+
+    def _is_valid_recipe(self, recipe:Any)->None:
+        """Validation check for 'recipe' variable from main constructor. Must 
+        be implemented by any child class."""
+        pass
+
+    def _is_valid_parameters(self, parameters:Any)->None:
+        """Validation check for 'parameters' variable from main constructor. 
+        Must be implemented by any child class."""
+        pass
+
+    def _is_valid_requirements(self, requirements:Any)->None:
+        """Validation check for 'requirements' variable from main constructor. 
+        Must be implemented by any child class."""
+        pass

core/base_rule.py

@@ -0,0 +1,81 @@
+
+"""
+This file contains the base MEOW rule defintion. This should be inherited from 
+for all rule instances.
+
+Author(s): David Marchant
+"""
+
+from typing import Any
+
+from core.base_pattern import BasePattern
+from core.base_recipe import BaseRecipe
+from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
+from core.correctness.validation import valid_string, check_type, \
+    check_implementation
+
+
+class BaseRule:
+    # A unique identifier for the rule
+    name:str
+    # A pattern to be used in rule triggering
+    pattern:BasePattern
+    # A recipe to be used in rule execution
+    recipe:BaseRecipe
+    # The string name of the pattern class that can be used to create this rule
+    pattern_type:str=""
+    # The string name of the recipe class that can be used to create this rule
+    recipe_type:str=""
+    def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
+        """BaseRule Constructor. This will check that any class inheriting 
+        from it implements its validation functions. It will then call these on
+        the input parameters."""
+        check_implementation(type(self)._is_valid_pattern, BaseRule)
+        check_implementation(type(self)._is_valid_recipe, BaseRule)
+        self.__check_types_set()
+        self._is_valid_name(name)
+        self.name = name
+        self._is_valid_pattern(pattern)
+        self.pattern = pattern
+        self._is_valid_recipe(recipe)
+        self.recipe = recipe
+        check_type(pattern, BasePattern, hint="BaseRule.pattern")
+        check_type(recipe, BaseRecipe, hint="BaseRule.recipe")
+        if pattern.recipe != recipe.name:
+            raise ValueError(f"Cannot create Rule {name}. Pattern "
+                f"{pattern.name} does not identify Recipe {recipe.name}. It "
+                f"uses {pattern.recipe}")
+
+    def __new__(cls, *args, **kwargs):
+        """A check that this base class is not instantiated itself, only 
+        inherited from"""
+        if cls is BaseRule:
+            msg = get_drt_imp_msg(BaseRule)
+            raise TypeError(msg)
+        return object.__new__(cls)
+
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_RULE_NAME_CHARS)
+
+    def _is_valid_pattern(self, pattern:Any)->None:
+        """Validation check for 'pattern' variable from main constructor. Must 
+        be implemented by any child class."""
+        pass
+
+    def _is_valid_recipe(self, recipe:Any)->None:
+        """Validation check for 'recipe' variable from main constructor. Must 
+        be implemented by any child class."""
+        pass
+
+    def __check_types_set(self)->None:
+        """Validation check that the self.pattern_type and self.recipe_type 
+        attributes have been set in a child class."""
+        if self.pattern_type == "":
+            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
+                "does not set a pattern_type.")
+        if self.recipe_type == "":
+            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
+                "does not set a recipe_type.")

core/meow.py

@@ -1,541 +0,0 @@
-
-"""
-This file contains the core MEOW defintions, used throughout this package. 
-It is intended that these base definitions are what should be inherited from in
-order to create an extendable framework for event-based scheduling and 
-processing.
-
-Author(s): David Marchant
-"""
-import inspect
-import itertools
-import sys
-
-from copy import deepcopy
-from typing import Any, Union, Tuple, Dict, List
-
-from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
-    VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
-    SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
-from core.correctness.validation import valid_string, check_type, \
-    check_implementation, valid_list, valid_dict
-from functionality.naming import generate_rule_id
-
-
-class BaseRecipe:
-    # A unique identifier for the recipe
-    name:str
-    # Actual code to run
-    recipe:Any
-    # Possible parameters that could be overridden by a Pattern 
-    parameters:Dict[str, Any]
-    # Additional configuration options
-    requirements:Dict[str, Any]
-    def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, 
-            requirements:Dict[str,Any]={}):
-        """BaseRecipe Constructor. This will check that any class inheriting 
-        from it implements its validation functions. It will then call these on
-        the input parameters."""
-        check_implementation(type(self)._is_valid_recipe, BaseRecipe)
-        check_implementation(type(self)._is_valid_parameters, BaseRecipe)
-        check_implementation(type(self)._is_valid_requirements, BaseRecipe)
-        self._is_valid_name(name)
-        self.name = name
-        self._is_valid_recipe(recipe)
-        self.recipe = recipe
-        self._is_valid_parameters(parameters)
-        self.parameters = parameters
-        self._is_valid_requirements(requirements)
-        self.requirements = requirements
-
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BaseRecipe:
-            msg = get_drt_imp_msg(BaseRecipe)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def _is_valid_name(self, name:str)->None:
-        """Validation check for 'name' variable from main constructor. Is 
-        automatically called during initialisation. This does not need to be 
-        overridden by child classes."""
-        valid_string(name, VALID_RECIPE_NAME_CHARS)
-
-    def _is_valid_recipe(self, recipe:Any)->None:
-        """Validation check for 'recipe' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def _is_valid_parameters(self, parameters:Any)->None:
-        """Validation check for 'parameters' variable from main constructor. 
-        Must be implemented by any child class."""
-        pass
-
-    def _is_valid_requirements(self, requirements:Any)->None:
-        """Validation check for 'requirements' variable from main constructor. 
-        Must be implemented by any child class."""
-        pass
-
-
-class BasePattern:
-    # A unique identifier for the pattern
-    name:str
-    # An identifier of a recipe
-    recipe:str
-    # Parameters to be overridden in the recipe
-    parameters:Dict[str,Any]
-    # Parameters showing the potential outputs of a recipe
-    outputs:Dict[str,Any]
-    # A collection of variables to be swept over for job scheduling
-    sweep:Dict[str,Any]
-
-    def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={}, 
-            outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}):
-        """BasePattern Constructor. This will check that any class inheriting 
-        from it implements its validation functions. It will then call these on
-        the input parameters."""
-        check_implementation(type(self)._is_valid_recipe, BasePattern)
-        check_implementation(type(self)._is_valid_parameters, BasePattern)
-        check_implementation(type(self)._is_valid_output, BasePattern)
-        self._is_valid_name(name)
-        self.name = name
-        self._is_valid_recipe(recipe)
-        self.recipe = recipe
-        self._is_valid_parameters(parameters)
-        self.parameters = parameters
-        self._is_valid_output(outputs)
-        self.outputs = outputs
-        self._is_valid_sweep(sweep)
-        self.sweep = sweep
-
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BasePattern:
-            msg = get_drt_imp_msg(BasePattern)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def _is_valid_name(self, name:str)->None:
-        """Validation check for 'name' variable from main constructor. Is 
-        automatically called during initialisation. This does not need to be 
-        overridden by child classes."""
-        valid_string(name, VALID_PATTERN_NAME_CHARS)
-
-    def _is_valid_recipe(self, recipe:Any)->None:
-        """Validation check for 'recipe' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def _is_valid_parameters(self, parameters:Any)->None:
-        """Validation check for 'parameters' variable from main constructor. 
-        Must be implemented by any child class."""
-        pass
-
-    def _is_valid_output(self, outputs:Any)->None:
-        """Validation check for 'outputs' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None:
-        """Validation check for 'sweep' variable from main constructor. This 
-        function is implemented to check for the types given in the signature, 
-        and must be overridden if these differ."""
-        check_type(sweep, Dict, hint="BasePattern.sweep")
-        if not sweep:
-            return
-        for _, v in sweep.items():
-            valid_dict(
-                v, str, Any, [
-                    SWEEP_START, SWEEP_STOP, SWEEP_JUMP
-                ], strict=True)
-
-            check_type(
-                v[SWEEP_START], 
-                expected_type=int, 
-                alt_types=[float, complex],
-                hint=f"BasePattern.sweep[{SWEEP_START}]"
-            )
-            check_type(
-                v[SWEEP_STOP], 
-                expected_type=int, 
-                alt_types=[float, complex],
-                hint=f"BasePattern.sweep[{SWEEP_STOP}]"
-            )
-            check_type(
-                v[SWEEP_JUMP], 
-                expected_type=int, 
-                alt_types=[float, complex],
-                hint=f"BasePattern.sweep[{SWEEP_JUMP}]"
-            )
-            # Try to check that this loop is not infinite
-            if v[SWEEP_JUMP] == 0:
-                raise ValueError(
-                    f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
-                )
-            elif v[SWEEP_JUMP] > 0:
-                if not v[SWEEP_STOP] > v[SWEEP_START]:
-                    raise ValueError(
-                        f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
-                        "value where the end point is smaller than the start."
-                    )
-            elif v[SWEEP_JUMP] < 0:
-                if not v[SWEEP_STOP] < v[SWEEP_START]:
-                    raise ValueError(
-                        f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
-                        "value where the end point is smaller than the start."
-                    )
-
-    def expand_sweeps(self)->List[Tuple[str,Any]]:
-        """Function to get all combinations of sweep parameters"""
-        values_dict = {}
-        # get a collection of a individual sweep values
-        for var, val in self.sweep.items():
-            values_dict[var] = []
-            par_val = val[SWEEP_START]
-            while par_val <= val[SWEEP_STOP]:
-                values_dict[var].append((var, par_val))
-                par_val += val[SWEEP_JUMP]
-
-        # combine all combinations of sweep values
-        return list(itertools.product(
-            *[v for v in values_dict.values()]))
-
-
-class BaseRule:
-    # A unique identifier for the rule
-    name:str
-    # A pattern to be used in rule triggering
-    pattern:BasePattern
-    # A recipe to be used in rule execution
-    recipe:BaseRecipe
-    # The string name of the pattern class that can be used to create this rule
-    pattern_type:str=""
-    # The string name of the recipe class that can be used to create this rule
-    recipe_type:str=""
-    def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
-        """BaseRule Constructor. This will check that any class inheriting 
-        from it implements its validation functions. It will then call these on
-        the input parameters."""
-        check_implementation(type(self)._is_valid_pattern, BaseRule)
-        check_implementation(type(self)._is_valid_recipe, BaseRule)
-        self.__check_types_set()
-        self._is_valid_name(name)
-        self.name = name
-        self._is_valid_pattern(pattern)
-        self.pattern = pattern
-        self._is_valid_recipe(recipe)
-        self.recipe = recipe
-        check_type(pattern, BasePattern, hint="BaseRule.pattern")
-        check_type(recipe, BaseRecipe, hint="BaseRule.recipe")
-        if pattern.recipe != recipe.name:
-            raise ValueError(f"Cannot create Rule {name}. Pattern "
-                f"{pattern.name} does not identify Recipe {recipe.name}. It "
-                f"uses {pattern.recipe}")
-
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BaseRule:
-            msg = get_drt_imp_msg(BaseRule)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def _is_valid_name(self, name:str)->None:
-        """Validation check for 'name' variable from main constructor. Is 
-        automatically called during initialisation. This does not need to be 
-        overridden by child classes."""
-        valid_string(name, VALID_RULE_NAME_CHARS)
-
-    def _is_valid_pattern(self, pattern:Any)->None:
-        """Validation check for 'pattern' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def _is_valid_recipe(self, recipe:Any)->None:
-        """Validation check for 'recipe' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def __check_types_set(self)->None:
-        """Validation check that the self.pattern_type and self.recipe_type 
-        attributes have been set in a child class."""
-        if self.pattern_type == "":
-            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
-                "does not set a pattern_type.")
-        if self.recipe_type == "":
-            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
-                "does not set a recipe_type.")
-
-
-class BaseMonitor:
-    # A collection of patterns
-    _patterns: Dict[str, BasePattern]
-    # A collection of recipes
-    _recipes: Dict[str, BaseRecipe]
-    # A collection of rules derived from _patterns and _recipes
-    _rules: Dict[str, BaseRule]
-    # A channel for sending messages to the runner. Note that this is not 
-    # initialised within the constructor, but within the runner when passed the
-    # monitor is passed to it.
-    to_runner: VALID_CHANNELS
-    def __init__(self, patterns:Dict[str,BasePattern], 
-            recipes:Dict[str,BaseRecipe])->None:
-        """BaseMonitor Constructor. This will check that any class inheriting 
-        from it implements its validation functions. It will then call these on
-        the input parameters."""
-        check_implementation(type(self).start, BaseMonitor)
-        check_implementation(type(self).stop, BaseMonitor)
-        check_implementation(type(self)._is_valid_patterns, BaseMonitor)
-        self._is_valid_patterns(patterns)
-        check_implementation(type(self)._is_valid_recipes, BaseMonitor)
-        self._is_valid_recipes(recipes)
-        check_implementation(type(self).add_pattern, BaseMonitor)
-        check_implementation(type(self).update_pattern, BaseMonitor)
-        check_implementation(type(self).remove_pattern, BaseMonitor)
-        check_implementation(type(self).get_patterns, BaseMonitor)
-        check_implementation(type(self).add_recipe, BaseMonitor)
-        check_implementation(type(self).update_recipe, BaseMonitor)
-        check_implementation(type(self).remove_recipe, BaseMonitor)
-        check_implementation(type(self).get_recipes, BaseMonitor)
-        check_implementation(type(self).get_rules, BaseMonitor)
-        # Ensure that patterns and recipes cannot be trivially modified from 
-        # outside the monitor, as this will cause internal consistency issues
-        self._patterns = deepcopy(patterns)
-        self._recipes = deepcopy(recipes)
-        self._rules = create_rules(patterns, recipes)
-        
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BaseMonitor:
-            msg = get_drt_imp_msg(BaseMonitor)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
-        """Validation check for 'patterns' variable from main constructor. Must
-        be implemented by any child class."""
-        pass
-
-    def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
-        """Validation check for 'recipes' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def start(self)->None:
-        """Function to start the monitor as an ongoing process/thread. Must be 
-        implemented by any child process"""
-        pass
-
-    def stop(self)->None:
-        """Function to stop the monitor as an ongoing process/thread. Must be 
-        implemented by any child process"""
-        pass
-
-    def add_pattern(self, pattern:BasePattern)->None:
-        """Function to add a pattern to the current definitions. Must be 
-        implemented by any child process."""
-        pass
-
-    def update_pattern(self, pattern:BasePattern)->None:
-        """Function to update a pattern in the current definitions. Must be 
-        implemented by any child process."""
-        pass
-
-    def remove_pattern(self, pattern:Union[str,BasePattern])->None:
-        """Function to remove a pattern from the current definitions. Must be 
-        implemented by any child process."""
-        pass
-
-    def get_patterns(self)->Dict[str,BasePattern]:
-        """Function to get a dictionary of all current pattern definitions. 
-        Must be implemented by any child process."""
-        pass
-
-    def add_recipe(self, recipe:BaseRecipe)->None:
-        """Function to add a recipe to the current definitions. Must be 
-        implemented by any child process."""
-        pass
-
-    def update_recipe(self, recipe:BaseRecipe)->None:
-        """Function to update a recipe in the current definitions. Must be 
-        implemented by any child process."""
-        pass
-
-    def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
-        """Function to remove a recipe from the current definitions. Must be 
-        implemented by any child process."""
-        pass
-
-    def get_recipes(self)->Dict[str,BaseRecipe]:
-        """Function to get a dictionary of all current recipe definitions. 
-        Must be implemented by any child process."""
-        pass
-
-    def get_rules(self)->Dict[str,BaseRule]:
-        """Function to get a dictionary of all current rule definitions. 
-        Must be implemented by any child process."""
-        pass
-
-
-class BaseHandler:
-    # A channel for sending messages to the runner. Note that this will be 
-    # overridden by a MeowRunner, if a handler instance is passed to it, and so
-    # does not need to be initialised within the handler itself.
-    to_runner: VALID_CHANNELS
-    # Directory where queued jobs are initially written to. Note that this 
-    # will be overridden by a MeowRunner, if a handler instance is passed to 
-    # it, and so does not need to be initialised within the handler itself.
-    job_queue_dir:str
-    def __init__(self)->None:
-        """BaseHandler Constructor. This will check that any class inheriting 
-        from it implements its validation functions."""
-        check_implementation(type(self).handle, BaseHandler)
-        check_implementation(type(self).valid_handle_criteria, BaseHandler)
-
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BaseHandler:
-            msg = get_drt_imp_msg(BaseHandler)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
-        """Function to determine given an event defintion, if this handler can 
-        process it or not. Must be implemented by any child process."""
-        pass
-
-    def handle(self, event:Dict[str,Any])->None:
-        """Function to handle a given event. Must be implemented by any child 
-        process."""
-        pass
-
-
-class BaseConductor:
-    # Directory where queued jobs are initially written to. Note that this 
-    # will be overridden by a MeowRunner, if a handler instance is passed to 
-    # it, and so does not need to be initialised within the handler itself.
-    job_queue_dir:str
-    # Directory where completed jobs are finally written to. Note that this 
-    # will be overridden by a MeowRunner, if a handler instance is passed to 
-    # it, and so does not need to be initialised within the handler itself.
-    job_output_dir:str
-    def __init__(self)->None:
-        """BaseConductor Constructor. This will check that any class inheriting
-        from it implements its validation functions."""
-        check_implementation(type(self).execute, BaseConductor)
-        check_implementation(type(self).valid_execute_criteria, BaseConductor)
-
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BaseConductor:
-            msg = get_drt_imp_msg(BaseConductor)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
-        """Function to determine given an job defintion, if this conductor can 
-        process it or not. Must be implemented by any child process."""
-        pass
-
-    def execute(self, job_dir:str)->None:
-        """Function to execute a given job directory. Must be implemented by 
-        any child process."""
-        pass
-
-
-def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], 
-        recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
-        new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
-    """Function to create any valid rules from a given collection of patterns 
-    and recipes. All inbuilt rule types are considered, with additional 
-    definitions provided through the 'new_rules' variable. Note that any 
-    provided pattern and recipe dictionaries must be keyed with the 
-    corresponding pattern and recipe names."""
-    # Validation of inputs
-    check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns")
-    check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes")
-    valid_list(new_rules, BaseRule, min_length=0)
-
-    # Convert a pattern list to a dictionary
-    if isinstance(patterns, list):
-        valid_list(patterns, BasePattern, min_length=0)
-        patterns = {pattern.name:pattern for pattern in patterns}
-    else:
-        # Validate the pattern dictionary
-        valid_dict(patterns, str, BasePattern, strict=False, min_length=0)
-        for k, v in patterns.items():
-            if k != v.name:
-                raise KeyError(
-                    f"Key '{k}' indexes unexpected Pattern '{v.name}' "
-                    "Pattern dictionaries must be keyed with the name of the "
-                    "Pattern.")
-
-    # Convert a recipe list into a dictionary
-    if isinstance(recipes, list):
-        valid_list(recipes, BaseRecipe, min_length=0)
-        recipes = {recipe.name:recipe for recipe in recipes}
-    else:
-        # Validate the recipe dictionary
-        valid_dict(recipes, str, BaseRecipe, strict=False, min_length=0)
-        for k, v in recipes.items():
-            if k != v.name:
-                raise KeyError(
-                    f"Key '{k}' indexes unexpected Recipe '{v.name}' "
-                    "Recipe dictionaries must be keyed with the name of the "
-                    "Recipe.")
-
-    # Try to create a rule for each rule in turn
-    generated_rules = {}
-    for pattern in patterns.values():
-        if pattern.recipe in recipes:
-            try:
-                rule = create_rule(pattern, recipes[pattern.recipe])
-                generated_rules[rule.name] = rule
-            except TypeError:
-                pass
-    return generated_rules
-
-def create_rule(pattern:BasePattern, recipe:BaseRecipe, 
-        new_rules:List[BaseRule]=[])->BaseRule:
-    """Function to create a valid rule from a given pattern and recipe. All 
-    inbuilt rule types are considered, with additional definitions provided 
-    through the 'new_rules' variable."""
-    check_type(pattern, BasePattern, hint="create_rule.pattern")
-    check_type(recipe, BaseRecipe, hint="create_rule.recipe")
-    valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules")
-
-    print("passed initial check")
-
-    # Imported here to avoid circular imports at top of file
-    import rules
-    # Get a dictionary of all inbuilt rules
-    all_rules ={(r.pattern_type, r.recipe_type):r for r in [r[1] \
-        for r in inspect.getmembers(sys.modules["rules"], inspect.isclass) \
-        if (issubclass(r[1], BaseRule))]}
-
-    print("got base rules")
-
-    # Add in new rules
-    for rule in new_rules:
-        all_rules[(rule.pattern_type, rule.recipe_type)] = rule
-
-    print("got new rules")
-
-    # Find appropriate rule type from pattern and recipe types
-    key = (type(pattern).__name__, type(recipe).__name__)
-    print("got key")
-    if (key) in all_rules:
-        return all_rules[key](
-            generate_rule_id(), 
-            pattern, 
-            recipe
-        )
-    print("no key")
-    # Raise error if not valid rule type can be found
-    raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe "
-        f"'{recipe}' could be found.")

core/runner.py

@@ -14,10 +14,12 @@ from multiprocessing import Pipe
 from random import randrange
 from typing import Any, Union, Dict, List
 
+from core.base_conductor import BaseConductor
+from core.base_handler import BaseHandler
+from core.base_monitor import BaseMonitor
 from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
     VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
 from core.correctness.validation import check_type, valid_list, valid_dir_path
-from core.meow import BaseHandler, BaseMonitor, BaseConductor
 from functionality.debug import setup_debugging, print_debug
 from functionality.file_io import make_dir, read_yaml
 from functionality.process_io import wait

functionality/meow.py

@@ -6,13 +6,17 @@ Author(s): David Marchant
 
 from datetime import datetime
 from os.path import basename, dirname, relpath, splitext
-from typing import Any, Dict
+from typing import Any, Dict, Union, List
 
+from core.base_pattern import BasePattern
+from core.base_recipe import BaseRecipe
+from core.base_rule import BaseRule
+from core.correctness.validation import check_type, valid_dict, valid_list
 from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
     EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \
     JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \
     STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH
-from functionality.naming import generate_job_id
+from functionality.naming import generate_job_id, generate_rule_id
 
 
 # mig trigger keyword replacements
@@ -101,3 +105,84 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
 
     return {**extras, **job_dict}
 
+def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], 
+        recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
+        new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
+    """Function to create any valid rules from a given collection of patterns 
+    and recipes. All inbuilt rule types are considered, with additional 
+    definitions provided through the 'new_rules' variable. Note that any 
+    provided pattern and recipe dictionaries must be keyed with the 
+    corresponding pattern and recipe names."""
+    # Validation of inputs
+    check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns")
+    check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes")
+    valid_list(new_rules, BaseRule, min_length=0)
+
+    # Convert a pattern list to a dictionary
+    if isinstance(patterns, list):
+        valid_list(patterns, BasePattern, min_length=0)
+        patterns = {pattern.name:pattern for pattern in patterns}
+    else:
+        # Validate the pattern dictionary
+        valid_dict(patterns, str, BasePattern, strict=False, min_length=0)
+        for k, v in patterns.items():
+            if k != v.name:
+                raise KeyError(
+                    f"Key '{k}' indexes unexpected Pattern '{v.name}' "
+                    "Pattern dictionaries must be keyed with the name of the "
+                    "Pattern.")
+
+    # Convert a recipe list into a dictionary
+    if isinstance(recipes, list):
+        valid_list(recipes, BaseRecipe, min_length=0)
+        recipes = {recipe.name:recipe for recipe in recipes}
+    else:
+        # Validate the recipe dictionary
+        valid_dict(recipes, str, BaseRecipe, strict=False, min_length=0)
+        for k, v in recipes.items():
+            if k != v.name:
+                raise KeyError(
+                    f"Key '{k}' indexes unexpected Recipe '{v.name}' "
+                    "Recipe dictionaries must be keyed with the name of the "
+                    "Recipe.")
+
+    # Try to create a rule for each rule in turn
+    generated_rules = {}
+    for pattern in patterns.values():
+        if pattern.recipe in recipes:
+            try:
+                rule = create_rule(pattern, recipes[pattern.recipe])
+                generated_rules[rule.name] = rule
+            except TypeError:
+                pass
+    return generated_rules
+
+def create_rule(pattern:BasePattern, recipe:BaseRecipe, 
+        new_rules:List[BaseRule]=[])->BaseRule:
+    """Function to create a valid rule from a given pattern and recipe. All 
+    inbuilt rule types are considered, with additional definitions provided 
+    through the 'new_rules' variable."""
+    check_type(pattern, BasePattern, hint="create_rule.pattern")
+    check_type(recipe, BaseRecipe, hint="create_rule.recipe")
+    valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules")
+
+    # TODO fix me
+    # Imported here to avoid circular imports at top of file
+    import rules
+    all_rules = {(r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()}
+
+    # Add in new rules
+    for rule in new_rules:
+        all_rules[(rule.pattern_type, rule.recipe_type)] = rule
+
+    # Find appropriate rule type from pattern and recipe types
+    key = (type(pattern).__name__, type(recipe).__name__)
+    if (key) in all_rules:
+        return all_rules[key](
+            generate_rule_id(), 
+            pattern, 
+            recipe
+        )
+    # Raise error if not valid rule type can be found
+    raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe "
+        f"'{recipe}' could be found.")

patterns/file_event_pattern.py

@@ -18,17 +18,19 @@ from typing import Any, Union, Dict, List
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 
+from core.base_recipe import BaseRecipe
+from core.base_monitor import BaseMonitor
+from core.base_pattern import BasePattern
+from core.base_rule import BaseRule
 from core.correctness.validation import check_type, valid_string, \
     valid_dict, valid_list, valid_path, valid_dir_path
 from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
     VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
     FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
     FILE_RETROACTIVE_EVENT, SHA256
-from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \
-    create_rule
 from functionality.debug import setup_debugging, print_debug
 from functionality.hashing import get_file_hash
-from functionality.meow import create_watchdog_event
+from functionality.meow import create_rule, create_watchdog_event
 
 # Events that are monitored by default
 _DEFAULT_MASK = [

recipes/jupyter_notebook_recipe.py

@@ -11,6 +11,8 @@ import sys
 
 from typing import Any, Tuple, Dict
 
+from core.base_recipe import BaseRecipe
+from core.base_handler import BaseHandler
 from core.correctness.validation import check_type, valid_string, \
     valid_dict, valid_path, valid_dir_path, valid_event
 from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
@@ -19,7 +21,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     JOB_ID, WATCHDOG_BASE, META_FILE, \
     PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
     EVENT_RULE, get_base_file
-from core.meow import BaseRecipe, BaseHandler
 from functionality.debug import setup_debugging, print_debug
 from functionality.file_io import make_dir, read_notebook, write_notebook, \
     write_yaml

recipes/python_recipe.py

@@ -10,6 +10,8 @@ import sys
 
 from typing import Any, Tuple, Dict, List
 
+from core.base_recipe import BaseRecipe
+from core.base_handler import BaseHandler
 from core.correctness.validation import check_script, valid_string, \
     valid_dict, valid_event, valid_dir_path
 from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
@@ -18,7 +20,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     JOB_ID, WATCHDOG_BASE, META_FILE, \
     PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
     get_base_file
-from core.meow import BaseRecipe, BaseHandler
 from functionality.debug import setup_debugging, print_debug
 from functionality.file_io import make_dir, read_file_lines, write_file, \
     write_yaml, lines_to_string

rules/file_event_jupyter_notebook_rule.py

@@ -5,8 +5,9 @@ and JupyterNotebookRecipe.
 
 Author(s): David Marchant
 """
+
+from core.base_rule import BaseRule
 from core.correctness.validation import check_type
-from core.meow import BaseRule
 from patterns.file_event_pattern import FileEventPattern
 from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
 

rules/file_event_python_rule.py

@@ -5,8 +5,9 @@ and PythonRecipe.
 
 Author(s): David Marchant
 """
+
+from core.base_rule import BaseRule
 from core.correctness.validation import check_type
-from core.meow import BaseRule
 from patterns.file_event_pattern import FileEventPattern
 from recipes.python_recipe import PythonRecipe
 

tests/shared.py

@@ -7,7 +7,8 @@ import os
 
 from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
 from functionality.file_io import make_dir, rmtree
-
+from patterns import FileEventPattern
+from recipes import JupyterNotebookRecipe
 
 # testing 
 TEST_DIR = "test_files"
@@ -15,6 +16,7 @@ TEST_MONITOR_BASE = "test_monitor_base"
 TEST_JOB_QUEUE = "test_job_queue_dir"
 TEST_JOB_OUTPUT = "test_job_output"
 
+
 def setup():
     make_dir(TEST_DIR, ensure_clean=True)
     make_dir(TEST_MONITOR_BASE, ensure_clean=True)
@@ -302,3 +304,13 @@ ADDING_NOTEBOOK = {
  "nbformat": 4,
  "nbformat_minor": 4
 }
+
+valid_pattern_one = FileEventPattern(
+    "pattern_one", "path_one", "recipe_one", "file_one")
+valid_pattern_two = FileEventPattern(
+    "pattern_two", "path_two", "recipe_two", "file_two")
+
+valid_recipe_one = JupyterNotebookRecipe(
+    "recipe_one", BAREBONES_NOTEBOOK)
+valid_recipe_two = JupyterNotebookRecipe(
+    "recipe_two", BAREBONES_NOTEBOOK)

tests/test_conductors.py

@@ -11,12 +11,11 @@ from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
     STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
     JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
     EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
-from core.meow import create_rule
 from conductors import LocalPythonConductor
 from functionality.file_io import read_file, read_yaml, write_file, \
     write_notebook, write_yaml, lines_to_string, make_dir
 from functionality.hashing import get_file_hash
-from functionality.meow import create_watchdog_event, create_job
+from functionality.meow import create_watchdog_event, create_job, create_rule
 from patterns import FileEventPattern
 from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
     papermill_job_func

tests/test_functionality.py

@@ -6,20 +6,21 @@ import os
 from datetime import datetime
 from multiprocessing import Pipe, Queue
 from time import sleep
+from typing import Dict
 
+from core.base_rule import BaseRule
 from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
     SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
     WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
     PYTHON_FUNC, JOB_ID, JOB_EVENT, \
     JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
     JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
-from core.meow import create_rule
 from functionality.file_io import lines_to_string, make_dir, read_file, \
     read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
     write_notebook, write_yaml    
 from functionality.hashing import get_file_hash
-from functionality.meow import create_event, create_job, \
+from functionality.meow import create_event, create_job, create_rule, \
-    create_watchdog_event, replace_keywords, \
+    create_rules, create_watchdog_event, replace_keywords, \
     KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
     KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
     KEYWORD_REL_PATH
@@ -29,8 +30,9 @@ from functionality.parameterisation import parameterize_jupyter_notebook, \
 from functionality.process_io import wait
 from patterns import FileEventPattern
 from recipes import JupyterNotebookRecipe
-from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \
+from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
-    APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
+    valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
+    COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
 
 class DebugTests(unittest.TestCase):
     def setUp(self)->None:
@@ -553,6 +555,58 @@ class MeowTests(unittest.TestCase):
         self.assertEqual(replaced["M"], "A") 
         self.assertEqual(replaced["N"], 1) 
 
+    # Test that create_rule creates a rule from pattern and recipe
+    def testCreateRule(self)->None:
+        rule = create_rule(valid_pattern_one, valid_recipe_one)
+
+        self.assertIsInstance(rule, BaseRule)
+
+        with self.assertRaises(ValueError):
+            rule = create_rule(valid_pattern_one, valid_recipe_two)
+    
+    # Test that create_rules creates nothing from nothing
+    def testCreateRulesMinimum(self)->None:
+        rules = create_rules({}, {})
+
+        self.assertEqual(len(rules), 0)
+
+    # Test that create_rules creates rules from patterns and recipes
+    def testCreateRulesPatternsAndRecipesDicts(self)->None:
+        patterns = {
+            valid_pattern_one.name: valid_pattern_one,
+            valid_pattern_two.name: valid_pattern_two
+        }
+        recipes = {
+            valid_recipe_one.name: valid_recipe_one,
+            valid_recipe_two.name: valid_recipe_two
+        }
+        rules = create_rules(patterns, recipes)
+        self.assertIsInstance(rules, Dict)
+        self.assertEqual(len(rules), 2)
+        for k, rule in rules.items():
+            self.assertIsInstance(k, str)
+            self.assertIsInstance(rule, BaseRule)
+            self.assertEqual(k, rule.name)
+
+    # Test that create_rules creates nothing from invalid pattern inputs
+    def testCreateRulesMisindexedPatterns(self)->None:
+        patterns = {
+            valid_pattern_two.name: valid_pattern_one,
+            valid_pattern_one.name: valid_pattern_two
+        }
+        with self.assertRaises(KeyError):
+            create_rules(patterns, {})
+
+    # Test that create_rules creates nothing from invalid recipe inputs
+    def testCreateRulesMisindexedRecipes(self)->None:
+        recipes = {
+            valid_recipe_two.name: valid_recipe_one,
+            valid_recipe_one.name: valid_recipe_two
+        }
+        with self.assertRaises(KeyError):
+            create_rules({}, recipes)
+
+
 
 class NamingTests(unittest.TestCase):
     def setUp(self)->None:

tests/test_meow.py

@@ -3,24 +3,18 @@ import unittest
  
 from typing import Any, Union, Tuple, Dict
 
+from core.base_conductor import BaseConductor
+from core.base_handler import BaseHandler
+from core.base_monitor import BaseMonitor
+from core.base_pattern import BasePattern
+from core.base_recipe import BaseRecipe
+from core.base_rule import BaseRule
 from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
-from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
-    BaseHandler, BaseConductor, create_rules, create_rule
 from patterns import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
+from shared import setup, teardown, valid_pattern_one, valid_recipe_one
-from shared import setup, teardown, BAREBONES_NOTEBOOK
-
-valid_pattern_one = FileEventPattern(
-    "pattern_one", "path_one", "recipe_one", "file_one")
-valid_pattern_two = FileEventPattern(
-    "pattern_two", "path_two", "recipe_two", "file_two")
-
-valid_recipe_one = JupyterNotebookRecipe(
-    "recipe_one", BAREBONES_NOTEBOOK)
-valid_recipe_two = JupyterNotebookRecipe(
-    "recipe_two", BAREBONES_NOTEBOOK)
 
 
+# TODO split me
 class MeowTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
@@ -162,57 +156,6 @@ class MeowTests(unittest.TestCase):
                 pass
         FullRule("name", valid_pattern_one, valid_recipe_one)
 
-    # Test that create_rule creates a rule from pattern and recipe
-    def testCreateRule(self)->None:
-        rule = create_rule(valid_pattern_one, valid_recipe_one)
-
-        self.assertIsInstance(rule, BaseRule)
-
-        with self.assertRaises(ValueError):
-            rule = create_rule(valid_pattern_one, valid_recipe_two)
-    
-    # Test that create_rules creates nothing from nothing
-    def testCreateRulesMinimum(self)->None:
-        rules = create_rules({}, {})
-
-        self.assertEqual(len(rules), 0)
-
-    # Test that create_rules creates rules from patterns and recipes
-    def testCreateRulesPatternsAndRecipesDicts(self)->None:
-        patterns = {
-            valid_pattern_one.name: valid_pattern_one,
-            valid_pattern_two.name: valid_pattern_two
-        }
-        recipes = {
-            valid_recipe_one.name: valid_recipe_one,
-            valid_recipe_two.name: valid_recipe_two
-        }
-        rules = create_rules(patterns, recipes)
-        self.assertIsInstance(rules, Dict)
-        self.assertEqual(len(rules), 2)
-        for k, rule in rules.items():
-            self.assertIsInstance(k, str)
-            self.assertIsInstance(rule, BaseRule)
-            self.assertEqual(k, rule.name)
-
-    # Test that create_rules creates nothing from invalid pattern inputs
-    def testCreateRulesMisindexedPatterns(self)->None:
-        patterns = {
-            valid_pattern_two.name: valid_pattern_one,
-            valid_pattern_one.name: valid_pattern_two
-        }
-        with self.assertRaises(KeyError):
-            create_rules(patterns, {})
-
-    # Test that create_rules creates nothing from invalid recipe inputs
-    def testCreateRulesMisindexedRecipes(self)->None:
-        recipes = {
-            valid_recipe_two.name: valid_recipe_one,
-            valid_recipe_one.name: valid_recipe_two
-        }
-        with self.assertRaises(KeyError):
-            create_rules({}, recipes)
-
     # Test that BaseMonitor instantiation
     def testBaseMonitor(self)->None:
         with self.assertRaises(TypeError):

tests/test_recipes.py

@@ -13,11 +13,11 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
     PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
     get_base_file, get_job_file, get_result_file
 from core.correctness.validation import valid_job
-from core.meow import create_rules, create_rule
 from functionality.file_io import lines_to_string, make_dir, read_yaml, \
     write_file, write_notebook, write_yaml
 from functionality.hashing import get_file_hash
-from functionality.meow import create_job, create_watchdog_event
+from functionality.meow import create_job, create_rules, create_rule, \
+    create_watchdog_event
 from patterns.file_event_pattern import FileEventPattern
 from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
     PapermillHandler, papermill_job_func

tests/test_runner.py

@@ -5,10 +5,12 @@ import unittest
  
 from time import sleep
 
+from core.base_conductor import BaseConductor
+from core.base_handler import BaseHandler
+from core.base_monitor import BaseMonitor
 from conductors import LocalPythonConductor
 from core.correctness.vars import get_result_file, \
     JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME
-from core.meow import BaseMonitor, BaseHandler, BaseConductor
 from core.runner import MeowRunner
 from functionality.file_io import make_dir, read_file, read_notebook, read_yaml
 from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern