From 6fab2f7a10a0d60d734c236d208301cae7af73b6 Mon Sep 17 00:00:00 2001 From: PatchOfScotland Date: Fri, 10 Feb 2023 16:23:24 +0100 Subject: [PATCH] also refactored core.meow into seperate files in hope that it'll help solve circular imports --- .gitignore | 2 +- conductors/local_python_conductor.py | 2 +- core/base_conductor.py | 46 ++ core/base_handler.py | 46 ++ core/base_monitor.py | 128 +++++ core/base_pattern.py | 141 ++++++ core/base_recipe.py | 68 +++ core/base_rule.py | 81 ++++ core/meow.py | 541 ---------------------- core/runner.py | 4 +- functionality/meow.py | 89 +++- patterns/file_event_pattern.py | 8 +- recipes/jupyter_notebook_recipe.py | 3 +- recipes/python_recipe.py | 3 +- rules/file_event_jupyter_notebook_rule.py | 3 +- rules/file_event_python_rule.py | 3 +- tests/shared.py | 14 +- tests/test_conductors.py | 3 +- tests/test_functionality.py | 64 ++- tests/test_meow.py | 73 +-- tests/test_recipes.py | 4 +- tests/test_runner.py | 4 +- 22 files changed, 701 insertions(+), 629 deletions(-) create mode 100644 core/base_conductor.py create mode 100644 core/base_handler.py create mode 100644 core/base_monitor.py create mode 100644 core/base_pattern.py create mode 100644 core/base_recipe.py create mode 100644 core/base_rule.py delete mode 100644 core/meow.py diff --git a/.gitignore b/.gitignore index 74dc052..b3ec55e 100644 --- a/.gitignore +++ b/.gitignore @@ -51,7 +51,7 @@ coverage.xml .hypothesis/ .pytest_cache/ tests/test_monitor_base -tests/test_job_queue_dir_dir_dir_dir_dir_dir_dir_dir_dir_dir +tests/test_job_queue_dir tests/test_job_output # Translations diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index dffecd7..074d5d9 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -11,12 +11,12 @@ import shutil from datetime import datetime from typing import Any, Tuple, Dict +from core.base_conductor import BaseConductor from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \ STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \ JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR from core.correctness.validation import valid_job, valid_dir_path -from core.meow import BaseConductor from functionality.file_io import make_dir, read_yaml, write_file, write_yaml class LocalPythonConductor(BaseConductor): diff --git a/core/base_conductor.py b/core/base_conductor.py new file mode 100644 index 0000000..a998f99 --- /dev/null +++ b/core/base_conductor.py @@ -0,0 +1,46 @@ + +""" +This file contains the base MEOW conductor defintion. This should be inherited +from for all conductor instances. + +Author(s): David Marchant +""" + +from typing import Any, Tuple, Dict + +from core.correctness.vars import get_drt_imp_msg +from core.correctness.validation import check_implementation + + +class BaseConductor: + # Directory where queued jobs are initially written to. Note that this + # will be overridden by a MeowRunner, if a handler instance is passed to + # it, and so does not need to be initialised within the handler itself. + job_queue_dir:str + # Directory where completed jobs are finally written to. Note that this + # will be overridden by a MeowRunner, if a handler instance is passed to + # it, and so does not need to be initialised within the handler itself. + job_output_dir:str + def __init__(self)->None: + """BaseConductor Constructor. This will check that any class inheriting + from it implements its validation functions.""" + check_implementation(type(self).execute, BaseConductor) + check_implementation(type(self).valid_execute_criteria, BaseConductor) + + def __new__(cls, *args, **kwargs): + """A check that this base class is not instantiated itself, only + inherited from""" + if cls is BaseConductor: + msg = get_drt_imp_msg(BaseConductor) + raise TypeError(msg) + return object.__new__(cls) + + def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: + """Function to determine given an job defintion, if this conductor can + process it or not. Must be implemented by any child process.""" + pass + + def execute(self, job_dir:str)->None: + """Function to execute a given job directory. Must be implemented by + any child process.""" + pass diff --git a/core/base_handler.py b/core/base_handler.py new file mode 100644 index 0000000..285f159 --- /dev/null +++ b/core/base_handler.py @@ -0,0 +1,46 @@ + +""" +This file contains the base MEOW handler defintion. This should be inherited +from for all handler instances. + +Author(s): David Marchant +""" + +from typing import Any, Tuple, Dict + +from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS +from core.correctness.validation import check_implementation + + +class BaseHandler: + # A channel for sending messages to the runner. Note that this will be + # overridden by a MeowRunner, if a handler instance is passed to it, and so + # does not need to be initialised within the handler itself. + to_runner: VALID_CHANNELS + # Directory where queued jobs are initially written to. Note that this + # will be overridden by a MeowRunner, if a handler instance is passed to + # it, and so does not need to be initialised within the handler itself. + job_queue_dir:str + def __init__(self)->None: + """BaseHandler Constructor. This will check that any class inheriting + from it implements its validation functions.""" + check_implementation(type(self).handle, BaseHandler) + check_implementation(type(self).valid_handle_criteria, BaseHandler) + + def __new__(cls, *args, **kwargs): + """A check that this base class is not instantiated itself, only + inherited from""" + if cls is BaseHandler: + msg = get_drt_imp_msg(BaseHandler) + raise TypeError(msg) + return object.__new__(cls) + + def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]: + """Function to determine given an event defintion, if this handler can + process it or not. Must be implemented by any child process.""" + pass + + def handle(self, event:Dict[str,Any])->None: + """Function to handle a given event. Must be implemented by any child + process.""" + pass diff --git a/core/base_monitor.py b/core/base_monitor.py new file mode 100644 index 0000000..e8e709c --- /dev/null +++ b/core/base_monitor.py @@ -0,0 +1,128 @@ + +""" +This file contains the base MEOW monitor defintion. This should be inherited +from for all monitor instances. + +Author(s): David Marchant +""" + +from copy import deepcopy +from typing import Union, Dict + +from core.base_pattern import BasePattern +from core.base_recipe import BaseRecipe +from core.base_rule import BaseRule +from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS +from core.correctness.validation import check_implementation +from functionality.meow import create_rules + + +class BaseMonitor: + # A collection of patterns + _patterns: Dict[str, BasePattern] + # A collection of recipes + _recipes: Dict[str, BaseRecipe] + # A collection of rules derived from _patterns and _recipes + _rules: Dict[str, BaseRule] + # A channel for sending messages to the runner. Note that this is not + # initialised within the constructor, but within the runner when passed the + # monitor is passed to it. + to_runner: VALID_CHANNELS + def __init__(self, patterns:Dict[str,BasePattern], + recipes:Dict[str,BaseRecipe])->None: + """BaseMonitor Constructor. This will check that any class inheriting + from it implements its validation functions. It will then call these on + the input parameters.""" + check_implementation(type(self).start, BaseMonitor) + check_implementation(type(self).stop, BaseMonitor) + check_implementation(type(self)._is_valid_patterns, BaseMonitor) + self._is_valid_patterns(patterns) + check_implementation(type(self)._is_valid_recipes, BaseMonitor) + self._is_valid_recipes(recipes) + check_implementation(type(self).add_pattern, BaseMonitor) + check_implementation(type(self).update_pattern, BaseMonitor) + check_implementation(type(self).remove_pattern, BaseMonitor) + check_implementation(type(self).get_patterns, BaseMonitor) + check_implementation(type(self).add_recipe, BaseMonitor) + check_implementation(type(self).update_recipe, BaseMonitor) + check_implementation(type(self).remove_recipe, BaseMonitor) + check_implementation(type(self).get_recipes, BaseMonitor) + check_implementation(type(self).get_rules, BaseMonitor) + # Ensure that patterns and recipes cannot be trivially modified from + # outside the monitor, as this will cause internal consistency issues + self._patterns = deepcopy(patterns) + self._recipes = deepcopy(recipes) + self._rules = create_rules(patterns, recipes) + + def __new__(cls, *args, **kwargs): + """A check that this base class is not instantiated itself, only + inherited from""" + if cls is BaseMonitor: + msg = get_drt_imp_msg(BaseMonitor) + raise TypeError(msg) + return object.__new__(cls) + + def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None: + """Validation check for 'patterns' variable from main constructor. Must + be implemented by any child class.""" + pass + + def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None: + """Validation check for 'recipes' variable from main constructor. Must + be implemented by any child class.""" + pass + + def start(self)->None: + """Function to start the monitor as an ongoing process/thread. Must be + implemented by any child process""" + pass + + def stop(self)->None: + """Function to stop the monitor as an ongoing process/thread. Must be + implemented by any child process""" + pass + + def add_pattern(self, pattern:BasePattern)->None: + """Function to add a pattern to the current definitions. Must be + implemented by any child process.""" + pass + + def update_pattern(self, pattern:BasePattern)->None: + """Function to update a pattern in the current definitions. Must be + implemented by any child process.""" + pass + + def remove_pattern(self, pattern:Union[str,BasePattern])->None: + """Function to remove a pattern from the current definitions. Must be + implemented by any child process.""" + pass + + def get_patterns(self)->Dict[str,BasePattern]: + """Function to get a dictionary of all current pattern definitions. + Must be implemented by any child process.""" + pass + + def add_recipe(self, recipe:BaseRecipe)->None: + """Function to add a recipe to the current definitions. Must be + implemented by any child process.""" + pass + + def update_recipe(self, recipe:BaseRecipe)->None: + """Function to update a recipe in the current definitions. Must be + implemented by any child process.""" + pass + + def remove_recipe(self, recipe:Union[str,BaseRecipe])->None: + """Function to remove a recipe from the current definitions. Must be + implemented by any child process.""" + pass + + def get_recipes(self)->Dict[str,BaseRecipe]: + """Function to get a dictionary of all current recipe definitions. + Must be implemented by any child process.""" + pass + + def get_rules(self)->Dict[str,BaseRule]: + """Function to get a dictionary of all current rule definitions. + Must be implemented by any child process.""" + pass diff --git a/core/base_pattern.py b/core/base_pattern.py new file mode 100644 index 0000000..9277605 --- /dev/null +++ b/core/base_pattern.py @@ -0,0 +1,141 @@ + +""" +This file contains the base MEOW pattern defintion. This should be inherited +from for all pattern instances. + +Author(s): David Marchant +""" + +import itertools + +from typing import Any, Union, Tuple, Dict, List + +from core.correctness.vars import get_drt_imp_msg, \ + VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP +from core.correctness.validation import valid_string, check_type, \ + check_implementation, valid_dict + + +class BasePattern: + # A unique identifier for the pattern + name:str + # An identifier of a recipe + recipe:str + # Parameters to be overridden in the recipe + parameters:Dict[str,Any] + # Parameters showing the potential outputs of a recipe + outputs:Dict[str,Any] + # A collection of variables to be swept over for job scheduling + sweep:Dict[str,Any] + + def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={}, + outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}): + """BasePattern Constructor. This will check that any class inheriting + from it implements its validation functions. It will then call these on + the input parameters.""" + check_implementation(type(self)._is_valid_recipe, BasePattern) + check_implementation(type(self)._is_valid_parameters, BasePattern) + check_implementation(type(self)._is_valid_output, BasePattern) + self._is_valid_name(name) + self.name = name + self._is_valid_recipe(recipe) + self.recipe = recipe + self._is_valid_parameters(parameters) + self.parameters = parameters + self._is_valid_output(outputs) + self.outputs = outputs + self._is_valid_sweep(sweep) + self.sweep = sweep + + def __new__(cls, *args, **kwargs): + """A check that this base class is not instantiated itself, only + inherited from""" + if cls is BasePattern: + msg = get_drt_imp_msg(BasePattern) + raise TypeError(msg) + return object.__new__(cls) + + def _is_valid_name(self, name:str)->None: + """Validation check for 'name' variable from main constructor. Is + automatically called during initialisation. This does not need to be + overridden by child classes.""" + valid_string(name, VALID_PATTERN_NAME_CHARS) + + def _is_valid_recipe(self, recipe:Any)->None: + """Validation check for 'recipe' variable from main constructor. Must + be implemented by any child class.""" + pass + + def _is_valid_parameters(self, parameters:Any)->None: + """Validation check for 'parameters' variable from main constructor. + Must be implemented by any child class.""" + pass + + def _is_valid_output(self, outputs:Any)->None: + """Validation check for 'outputs' variable from main constructor. Must + be implemented by any child class.""" + pass + + def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None: + """Validation check for 'sweep' variable from main constructor. This + function is implemented to check for the types given in the signature, + and must be overridden if these differ.""" + check_type(sweep, Dict, hint="BasePattern.sweep") + if not sweep: + return + for _, v in sweep.items(): + valid_dict( + v, str, Any, [ + SWEEP_START, SWEEP_STOP, SWEEP_JUMP + ], strict=True) + + check_type( + v[SWEEP_START], + expected_type=int, + alt_types=[float, complex], + hint=f"BasePattern.sweep[{SWEEP_START}]" + ) + check_type( + v[SWEEP_STOP], + expected_type=int, + alt_types=[float, complex], + hint=f"BasePattern.sweep[{SWEEP_STOP}]" + ) + check_type( + v[SWEEP_JUMP], + expected_type=int, + alt_types=[float, complex], + hint=f"BasePattern.sweep[{SWEEP_JUMP}]" + ) + # Try to check that this loop is not infinite + if v[SWEEP_JUMP] == 0: + raise ValueError( + f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero" + ) + elif v[SWEEP_JUMP] > 0: + if not v[SWEEP_STOP] > v[SWEEP_START]: + raise ValueError( + f"Cannot create sweep with a positive '{SWEEP_JUMP}' " + "value where the end point is smaller than the start." + ) + elif v[SWEEP_JUMP] < 0: + if not v[SWEEP_STOP] < v[SWEEP_START]: + raise ValueError( + f"Cannot create sweep with a negative '{SWEEP_JUMP}' " + "value where the end point is smaller than the start." + ) + + def expand_sweeps(self)->List[Tuple[str,Any]]: + """Function to get all combinations of sweep parameters""" + values_dict = {} + # get a collection of a individual sweep values + for var, val in self.sweep.items(): + values_dict[var] = [] + par_val = val[SWEEP_START] + while par_val <= val[SWEEP_STOP]: + values_dict[var].append((var, par_val)) + par_val += val[SWEEP_JUMP] + + # combine all combinations of sweep values + return list(itertools.product( + *[v for v in values_dict.values()])) diff --git a/core/base_recipe.py b/core/base_recipe.py new file mode 100644 index 0000000..b91b452 --- /dev/null +++ b/core/base_recipe.py @@ -0,0 +1,68 @@ + +""" +This file contains the base MEOW recipe defintion. This should be inherited +from for all recipe instances. + +Author(s): David Marchant +""" + +from typing import Any, Dict + +from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS +from core.correctness.validation import valid_string, check_implementation + + +class BaseRecipe: + # A unique identifier for the recipe + name:str + # Actual code to run + recipe:Any + # Possible parameters that could be overridden by a Pattern + parameters:Dict[str, Any] + # Additional configuration options + requirements:Dict[str, Any] + def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, + requirements:Dict[str,Any]={}): + """BaseRecipe Constructor. This will check that any class inheriting + from it implements its validation functions. It will then call these on + the input parameters.""" + check_implementation(type(self)._is_valid_recipe, BaseRecipe) + check_implementation(type(self)._is_valid_parameters, BaseRecipe) + check_implementation(type(self)._is_valid_requirements, BaseRecipe) + self._is_valid_name(name) + self.name = name + self._is_valid_recipe(recipe) + self.recipe = recipe + self._is_valid_parameters(parameters) + self.parameters = parameters + self._is_valid_requirements(requirements) + self.requirements = requirements + + def __new__(cls, *args, **kwargs): + """A check that this base class is not instantiated itself, only + inherited from""" + if cls is BaseRecipe: + msg = get_drt_imp_msg(BaseRecipe) + raise TypeError(msg) + return object.__new__(cls) + + def _is_valid_name(self, name:str)->None: + """Validation check for 'name' variable from main constructor. Is + automatically called during initialisation. This does not need to be + overridden by child classes.""" + valid_string(name, VALID_RECIPE_NAME_CHARS) + + def _is_valid_recipe(self, recipe:Any)->None: + """Validation check for 'recipe' variable from main constructor. Must + be implemented by any child class.""" + pass + + def _is_valid_parameters(self, parameters:Any)->None: + """Validation check for 'parameters' variable from main constructor. + Must be implemented by any child class.""" + pass + + def _is_valid_requirements(self, requirements:Any)->None: + """Validation check for 'requirements' variable from main constructor. + Must be implemented by any child class.""" + pass diff --git a/core/base_rule.py b/core/base_rule.py new file mode 100644 index 0000000..6119f56 --- /dev/null +++ b/core/base_rule.py @@ -0,0 +1,81 @@ + +""" +This file contains the base MEOW rule defintion. This should be inherited from +for all rule instances. + +Author(s): David Marchant +""" + +from typing import Any + +from core.base_pattern import BasePattern +from core.base_recipe import BaseRecipe +from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS +from core.correctness.validation import valid_string, check_type, \ + check_implementation + + +class BaseRule: + # A unique identifier for the rule + name:str + # A pattern to be used in rule triggering + pattern:BasePattern + # A recipe to be used in rule execution + recipe:BaseRecipe + # The string name of the pattern class that can be used to create this rule + pattern_type:str="" + # The string name of the recipe class that can be used to create this rule + recipe_type:str="" + def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe): + """BaseRule Constructor. This will check that any class inheriting + from it implements its validation functions. It will then call these on + the input parameters.""" + check_implementation(type(self)._is_valid_pattern, BaseRule) + check_implementation(type(self)._is_valid_recipe, BaseRule) + self.__check_types_set() + self._is_valid_name(name) + self.name = name + self._is_valid_pattern(pattern) + self.pattern = pattern + self._is_valid_recipe(recipe) + self.recipe = recipe + check_type(pattern, BasePattern, hint="BaseRule.pattern") + check_type(recipe, BaseRecipe, hint="BaseRule.recipe") + if pattern.recipe != recipe.name: + raise ValueError(f"Cannot create Rule {name}. Pattern " + f"{pattern.name} does not identify Recipe {recipe.name}. It " + f"uses {pattern.recipe}") + + def __new__(cls, *args, **kwargs): + """A check that this base class is not instantiated itself, only + inherited from""" + if cls is BaseRule: + msg = get_drt_imp_msg(BaseRule) + raise TypeError(msg) + return object.__new__(cls) + + def _is_valid_name(self, name:str)->None: + """Validation check for 'name' variable from main constructor. Is + automatically called during initialisation. This does not need to be + overridden by child classes.""" + valid_string(name, VALID_RULE_NAME_CHARS) + + def _is_valid_pattern(self, pattern:Any)->None: + """Validation check for 'pattern' variable from main constructor. Must + be implemented by any child class.""" + pass + + def _is_valid_recipe(self, recipe:Any)->None: + """Validation check for 'recipe' variable from main constructor. Must + be implemented by any child class.""" + pass + + def __check_types_set(self)->None: + """Validation check that the self.pattern_type and self.recipe_type + attributes have been set in a child class.""" + if self.pattern_type == "": + raise AttributeError(f"Rule Class '{self.__class__.__name__}' " + "does not set a pattern_type.") + if self.recipe_type == "": + raise AttributeError(f"Rule Class '{self.__class__.__name__}' " + "does not set a recipe_type.") diff --git a/core/meow.py b/core/meow.py deleted file mode 100644 index 9b9fb0f..0000000 --- a/core/meow.py +++ /dev/null @@ -1,541 +0,0 @@ - -""" -This file contains the core MEOW defintions, used throughout this package. -It is intended that these base definitions are what should be inherited from in -order to create an extendable framework for event-based scheduling and -processing. - -Author(s): David Marchant -""" -import inspect -import itertools -import sys - -from copy import deepcopy -from typing import Any, Union, Tuple, Dict, List - -from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ - VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \ - SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg -from core.correctness.validation import valid_string, check_type, \ - check_implementation, valid_list, valid_dict -from functionality.naming import generate_rule_id - - -class BaseRecipe: - # A unique identifier for the recipe - name:str - # Actual code to run - recipe:Any - # Possible parameters that could be overridden by a Pattern - parameters:Dict[str, Any] - # Additional configuration options - requirements:Dict[str, Any] - def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, - requirements:Dict[str,Any]={}): - """BaseRecipe Constructor. This will check that any class inheriting - from it implements its validation functions. It will then call these on - the input parameters.""" - check_implementation(type(self)._is_valid_recipe, BaseRecipe) - check_implementation(type(self)._is_valid_parameters, BaseRecipe) - check_implementation(type(self)._is_valid_requirements, BaseRecipe) - self._is_valid_name(name) - self.name = name - self._is_valid_recipe(recipe) - self.recipe = recipe - self._is_valid_parameters(parameters) - self.parameters = parameters - self._is_valid_requirements(requirements) - self.requirements = requirements - - def __new__(cls, *args, **kwargs): - """A check that this base class is not instantiated itself, only - inherited from""" - if cls is BaseRecipe: - msg = get_drt_imp_msg(BaseRecipe) - raise TypeError(msg) - return object.__new__(cls) - - def _is_valid_name(self, name:str)->None: - """Validation check for 'name' variable from main constructor. Is - automatically called during initialisation. This does not need to be - overridden by child classes.""" - valid_string(name, VALID_RECIPE_NAME_CHARS) - - def _is_valid_recipe(self, recipe:Any)->None: - """Validation check for 'recipe' variable from main constructor. Must - be implemented by any child class.""" - pass - - def _is_valid_parameters(self, parameters:Any)->None: - """Validation check for 'parameters' variable from main constructor. - Must be implemented by any child class.""" - pass - - def _is_valid_requirements(self, requirements:Any)->None: - """Validation check for 'requirements' variable from main constructor. - Must be implemented by any child class.""" - pass - - -class BasePattern: - # A unique identifier for the pattern - name:str - # An identifier of a recipe - recipe:str - # Parameters to be overridden in the recipe - parameters:Dict[str,Any] - # Parameters showing the potential outputs of a recipe - outputs:Dict[str,Any] - # A collection of variables to be swept over for job scheduling - sweep:Dict[str,Any] - - def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={}, - outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}): - """BasePattern Constructor. This will check that any class inheriting - from it implements its validation functions. It will then call these on - the input parameters.""" - check_implementation(type(self)._is_valid_recipe, BasePattern) - check_implementation(type(self)._is_valid_parameters, BasePattern) - check_implementation(type(self)._is_valid_output, BasePattern) - self._is_valid_name(name) - self.name = name - self._is_valid_recipe(recipe) - self.recipe = recipe - self._is_valid_parameters(parameters) - self.parameters = parameters - self._is_valid_output(outputs) - self.outputs = outputs - self._is_valid_sweep(sweep) - self.sweep = sweep - - def __new__(cls, *args, **kwargs): - """A check that this base class is not instantiated itself, only - inherited from""" - if cls is BasePattern: - msg = get_drt_imp_msg(BasePattern) - raise TypeError(msg) - return object.__new__(cls) - - def _is_valid_name(self, name:str)->None: - """Validation check for 'name' variable from main constructor. Is - automatically called during initialisation. This does not need to be - overridden by child classes.""" - valid_string(name, VALID_PATTERN_NAME_CHARS) - - def _is_valid_recipe(self, recipe:Any)->None: - """Validation check for 'recipe' variable from main constructor. Must - be implemented by any child class.""" - pass - - def _is_valid_parameters(self, parameters:Any)->None: - """Validation check for 'parameters' variable from main constructor. - Must be implemented by any child class.""" - pass - - def _is_valid_output(self, outputs:Any)->None: - """Validation check for 'outputs' variable from main constructor. Must - be implemented by any child class.""" - pass - - def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None: - """Validation check for 'sweep' variable from main constructor. This - function is implemented to check for the types given in the signature, - and must be overridden if these differ.""" - check_type(sweep, Dict, hint="BasePattern.sweep") - if not sweep: - return - for _, v in sweep.items(): - valid_dict( - v, str, Any, [ - SWEEP_START, SWEEP_STOP, SWEEP_JUMP - ], strict=True) - - check_type( - v[SWEEP_START], - expected_type=int, - alt_types=[float, complex], - hint=f"BasePattern.sweep[{SWEEP_START}]" - ) - check_type( - v[SWEEP_STOP], - expected_type=int, - alt_types=[float, complex], - hint=f"BasePattern.sweep[{SWEEP_STOP}]" - ) - check_type( - v[SWEEP_JUMP], - expected_type=int, - alt_types=[float, complex], - hint=f"BasePattern.sweep[{SWEEP_JUMP}]" - ) - # Try to check that this loop is not infinite - if v[SWEEP_JUMP] == 0: - raise ValueError( - f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero" - ) - elif v[SWEEP_JUMP] > 0: - if not v[SWEEP_STOP] > v[SWEEP_START]: - raise ValueError( - f"Cannot create sweep with a positive '{SWEEP_JUMP}' " - "value where the end point is smaller than the start." - ) - elif v[SWEEP_JUMP] < 0: - if not v[SWEEP_STOP] < v[SWEEP_START]: - raise ValueError( - f"Cannot create sweep with a negative '{SWEEP_JUMP}' " - "value where the end point is smaller than the start." - ) - - def expand_sweeps(self)->List[Tuple[str,Any]]: - """Function to get all combinations of sweep parameters""" - values_dict = {} - # get a collection of a individual sweep values - for var, val in self.sweep.items(): - values_dict[var] = [] - par_val = val[SWEEP_START] - while par_val <= val[SWEEP_STOP]: - values_dict[var].append((var, par_val)) - par_val += val[SWEEP_JUMP] - - # combine all combinations of sweep values - return list(itertools.product( - *[v for v in values_dict.values()])) - - -class BaseRule: - # A unique identifier for the rule - name:str - # A pattern to be used in rule triggering - pattern:BasePattern - # A recipe to be used in rule execution - recipe:BaseRecipe - # The string name of the pattern class that can be used to create this rule - pattern_type:str="" - # The string name of the recipe class that can be used to create this rule - recipe_type:str="" - def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe): - """BaseRule Constructor. This will check that any class inheriting - from it implements its validation functions. It will then call these on - the input parameters.""" - check_implementation(type(self)._is_valid_pattern, BaseRule) - check_implementation(type(self)._is_valid_recipe, BaseRule) - self.__check_types_set() - self._is_valid_name(name) - self.name = name - self._is_valid_pattern(pattern) - self.pattern = pattern - self._is_valid_recipe(recipe) - self.recipe = recipe - check_type(pattern, BasePattern, hint="BaseRule.pattern") - check_type(recipe, BaseRecipe, hint="BaseRule.recipe") - if pattern.recipe != recipe.name: - raise ValueError(f"Cannot create Rule {name}. Pattern " - f"{pattern.name} does not identify Recipe {recipe.name}. It " - f"uses {pattern.recipe}") - - def __new__(cls, *args, **kwargs): - """A check that this base class is not instantiated itself, only - inherited from""" - if cls is BaseRule: - msg = get_drt_imp_msg(BaseRule) - raise TypeError(msg) - return object.__new__(cls) - - def _is_valid_name(self, name:str)->None: - """Validation check for 'name' variable from main constructor. Is - automatically called during initialisation. This does not need to be - overridden by child classes.""" - valid_string(name, VALID_RULE_NAME_CHARS) - - def _is_valid_pattern(self, pattern:Any)->None: - """Validation check for 'pattern' variable from main constructor. Must - be implemented by any child class.""" - pass - - def _is_valid_recipe(self, recipe:Any)->None: - """Validation check for 'recipe' variable from main constructor. Must - be implemented by any child class.""" - pass - - def __check_types_set(self)->None: - """Validation check that the self.pattern_type and self.recipe_type - attributes have been set in a child class.""" - if self.pattern_type == "": - raise AttributeError(f"Rule Class '{self.__class__.__name__}' " - "does not set a pattern_type.") - if self.recipe_type == "": - raise AttributeError(f"Rule Class '{self.__class__.__name__}' " - "does not set a recipe_type.") - - -class BaseMonitor: - # A collection of patterns - _patterns: Dict[str, BasePattern] - # A collection of recipes - _recipes: Dict[str, BaseRecipe] - # A collection of rules derived from _patterns and _recipes - _rules: Dict[str, BaseRule] - # A channel for sending messages to the runner. Note that this is not - # initialised within the constructor, but within the runner when passed the - # monitor is passed to it. - to_runner: VALID_CHANNELS - def __init__(self, patterns:Dict[str,BasePattern], - recipes:Dict[str,BaseRecipe])->None: - """BaseMonitor Constructor. This will check that any class inheriting - from it implements its validation functions. It will then call these on - the input parameters.""" - check_implementation(type(self).start, BaseMonitor) - check_implementation(type(self).stop, BaseMonitor) - check_implementation(type(self)._is_valid_patterns, BaseMonitor) - self._is_valid_patterns(patterns) - check_implementation(type(self)._is_valid_recipes, BaseMonitor) - self._is_valid_recipes(recipes) - check_implementation(type(self).add_pattern, BaseMonitor) - check_implementation(type(self).update_pattern, BaseMonitor) - check_implementation(type(self).remove_pattern, BaseMonitor) - check_implementation(type(self).get_patterns, BaseMonitor) - check_implementation(type(self).add_recipe, BaseMonitor) - check_implementation(type(self).update_recipe, BaseMonitor) - check_implementation(type(self).remove_recipe, BaseMonitor) - check_implementation(type(self).get_recipes, BaseMonitor) - check_implementation(type(self).get_rules, BaseMonitor) - # Ensure that patterns and recipes cannot be trivially modified from - # outside the monitor, as this will cause internal consistency issues - self._patterns = deepcopy(patterns) - self._recipes = deepcopy(recipes) - self._rules = create_rules(patterns, recipes) - - def __new__(cls, *args, **kwargs): - """A check that this base class is not instantiated itself, only - inherited from""" - if cls is BaseMonitor: - msg = get_drt_imp_msg(BaseMonitor) - raise TypeError(msg) - return object.__new__(cls) - - def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None: - """Validation check for 'patterns' variable from main constructor. Must - be implemented by any child class.""" - pass - - def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None: - """Validation check for 'recipes' variable from main constructor. Must - be implemented by any child class.""" - pass - - def start(self)->None: - """Function to start the monitor as an ongoing process/thread. Must be - implemented by any child process""" - pass - - def stop(self)->None: - """Function to stop the monitor as an ongoing process/thread. Must be - implemented by any child process""" - pass - - def add_pattern(self, pattern:BasePattern)->None: - """Function to add a pattern to the current definitions. Must be - implemented by any child process.""" - pass - - def update_pattern(self, pattern:BasePattern)->None: - """Function to update a pattern in the current definitions. Must be - implemented by any child process.""" - pass - - def remove_pattern(self, pattern:Union[str,BasePattern])->None: - """Function to remove a pattern from the current definitions. Must be - implemented by any child process.""" - pass - - def get_patterns(self)->Dict[str,BasePattern]: - """Function to get a dictionary of all current pattern definitions. - Must be implemented by any child process.""" - pass - - def add_recipe(self, recipe:BaseRecipe)->None: - """Function to add a recipe to the current definitions. Must be - implemented by any child process.""" - pass - - def update_recipe(self, recipe:BaseRecipe)->None: - """Function to update a recipe in the current definitions. Must be - implemented by any child process.""" - pass - - def remove_recipe(self, recipe:Union[str,BaseRecipe])->None: - """Function to remove a recipe from the current definitions. Must be - implemented by any child process.""" - pass - - def get_recipes(self)->Dict[str,BaseRecipe]: - """Function to get a dictionary of all current recipe definitions. - Must be implemented by any child process.""" - pass - - def get_rules(self)->Dict[str,BaseRule]: - """Function to get a dictionary of all current rule definitions. - Must be implemented by any child process.""" - pass - - -class BaseHandler: - # A channel for sending messages to the runner. Note that this will be - # overridden by a MeowRunner, if a handler instance is passed to it, and so - # does not need to be initialised within the handler itself. - to_runner: VALID_CHANNELS - # Directory where queued jobs are initially written to. Note that this - # will be overridden by a MeowRunner, if a handler instance is passed to - # it, and so does not need to be initialised within the handler itself. - job_queue_dir:str - def __init__(self)->None: - """BaseHandler Constructor. This will check that any class inheriting - from it implements its validation functions.""" - check_implementation(type(self).handle, BaseHandler) - check_implementation(type(self).valid_handle_criteria, BaseHandler) - - def __new__(cls, *args, **kwargs): - """A check that this base class is not instantiated itself, only - inherited from""" - if cls is BaseHandler: - msg = get_drt_imp_msg(BaseHandler) - raise TypeError(msg) - return object.__new__(cls) - - def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]: - """Function to determine given an event defintion, if this handler can - process it or not. Must be implemented by any child process.""" - pass - - def handle(self, event:Dict[str,Any])->None: - """Function to handle a given event. Must be implemented by any child - process.""" - pass - - -class BaseConductor: - # Directory where queued jobs are initially written to. Note that this - # will be overridden by a MeowRunner, if a handler instance is passed to - # it, and so does not need to be initialised within the handler itself. - job_queue_dir:str - # Directory where completed jobs are finally written to. Note that this - # will be overridden by a MeowRunner, if a handler instance is passed to - # it, and so does not need to be initialised within the handler itself. - job_output_dir:str - def __init__(self)->None: - """BaseConductor Constructor. This will check that any class inheriting - from it implements its validation functions.""" - check_implementation(type(self).execute, BaseConductor) - check_implementation(type(self).valid_execute_criteria, BaseConductor) - - def __new__(cls, *args, **kwargs): - """A check that this base class is not instantiated itself, only - inherited from""" - if cls is BaseConductor: - msg = get_drt_imp_msg(BaseConductor) - raise TypeError(msg) - return object.__new__(cls) - - def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: - """Function to determine given an job defintion, if this conductor can - process it or not. Must be implemented by any child process.""" - pass - - def execute(self, job_dir:str)->None: - """Function to execute a given job directory. Must be implemented by - any child process.""" - pass - - -def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], - recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]], - new_rules:List[BaseRule]=[])->Dict[str,BaseRule]: - """Function to create any valid rules from a given collection of patterns - and recipes. All inbuilt rule types are considered, with additional - definitions provided through the 'new_rules' variable. Note that any - provided pattern and recipe dictionaries must be keyed with the - corresponding pattern and recipe names.""" - # Validation of inputs - check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns") - check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes") - valid_list(new_rules, BaseRule, min_length=0) - - # Convert a pattern list to a dictionary - if isinstance(patterns, list): - valid_list(patterns, BasePattern, min_length=0) - patterns = {pattern.name:pattern for pattern in patterns} - else: - # Validate the pattern dictionary - valid_dict(patterns, str, BasePattern, strict=False, min_length=0) - for k, v in patterns.items(): - if k != v.name: - raise KeyError( - f"Key '{k}' indexes unexpected Pattern '{v.name}' " - "Pattern dictionaries must be keyed with the name of the " - "Pattern.") - - # Convert a recipe list into a dictionary - if isinstance(recipes, list): - valid_list(recipes, BaseRecipe, min_length=0) - recipes = {recipe.name:recipe for recipe in recipes} - else: - # Validate the recipe dictionary - valid_dict(recipes, str, BaseRecipe, strict=False, min_length=0) - for k, v in recipes.items(): - if k != v.name: - raise KeyError( - f"Key '{k}' indexes unexpected Recipe '{v.name}' " - "Recipe dictionaries must be keyed with the name of the " - "Recipe.") - - # Try to create a rule for each rule in turn - generated_rules = {} - for pattern in patterns.values(): - if pattern.recipe in recipes: - try: - rule = create_rule(pattern, recipes[pattern.recipe]) - generated_rules[rule.name] = rule - except TypeError: - pass - return generated_rules - -def create_rule(pattern:BasePattern, recipe:BaseRecipe, - new_rules:List[BaseRule]=[])->BaseRule: - """Function to create a valid rule from a given pattern and recipe. All - inbuilt rule types are considered, with additional definitions provided - through the 'new_rules' variable.""" - check_type(pattern, BasePattern, hint="create_rule.pattern") - check_type(recipe, BaseRecipe, hint="create_rule.recipe") - valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules") - - print("passed initial check") - - # Imported here to avoid circular imports at top of file - import rules - # Get a dictionary of all inbuilt rules - all_rules ={(r.pattern_type, r.recipe_type):r for r in [r[1] \ - for r in inspect.getmembers(sys.modules["rules"], inspect.isclass) \ - if (issubclass(r[1], BaseRule))]} - - print("got base rules") - - # Add in new rules - for rule in new_rules: - all_rules[(rule.pattern_type, rule.recipe_type)] = rule - - print("got new rules") - - # Find appropriate rule type from pattern and recipe types - key = (type(pattern).__name__, type(recipe).__name__) - print("got key") - if (key) in all_rules: - return all_rules[key]( - generate_rule_id(), - pattern, - recipe - ) - print("no key") - # Raise error if not valid rule type can be found - raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe " - f"'{recipe}' could be found.") diff --git a/core/runner.py b/core/runner.py index 9ae48db..e69def4 100644 --- a/core/runner.py +++ b/core/runner.py @@ -14,10 +14,12 @@ from multiprocessing import Pipe from random import randrange from typing import Any, Union, Dict, List +from core.base_conductor import BaseConductor +from core.base_handler import BaseHandler +from core.base_monitor import BaseMonitor from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \ VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR from core.correctness.validation import check_type, valid_list, valid_dir_path -from core.meow import BaseHandler, BaseMonitor, BaseConductor from functionality.debug import setup_debugging, print_debug from functionality.file_io import make_dir, read_yaml from functionality.process_io import wait diff --git a/functionality/meow.py b/functionality/meow.py index 4a310b9..e299cff 100644 --- a/functionality/meow.py +++ b/functionality/meow.py @@ -6,13 +6,17 @@ Author(s): David Marchant from datetime import datetime from os.path import basename, dirname, relpath, splitext -from typing import Any, Dict +from typing import Any, Dict, Union, List +from core.base_pattern import BasePattern +from core.base_recipe import BaseRecipe +from core.base_rule import BaseRule +from core.correctness.validation import check_type, valid_dict, valid_list from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \ JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \ STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH -from functionality.naming import generate_job_id +from functionality.naming import generate_job_id, generate_rule_id # mig trigger keyword replacements @@ -101,3 +105,84 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={} return {**extras, **job_dict} +def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], + recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]], + new_rules:List[BaseRule]=[])->Dict[str,BaseRule]: + """Function to create any valid rules from a given collection of patterns + and recipes. All inbuilt rule types are considered, with additional + definitions provided through the 'new_rules' variable. Note that any + provided pattern and recipe dictionaries must be keyed with the + corresponding pattern and recipe names.""" + # Validation of inputs + check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns") + check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes") + valid_list(new_rules, BaseRule, min_length=0) + + # Convert a pattern list to a dictionary + if isinstance(patterns, list): + valid_list(patterns, BasePattern, min_length=0) + patterns = {pattern.name:pattern for pattern in patterns} + else: + # Validate the pattern dictionary + valid_dict(patterns, str, BasePattern, strict=False, min_length=0) + for k, v in patterns.items(): + if k != v.name: + raise KeyError( + f"Key '{k}' indexes unexpected Pattern '{v.name}' " + "Pattern dictionaries must be keyed with the name of the " + "Pattern.") + + # Convert a recipe list into a dictionary + if isinstance(recipes, list): + valid_list(recipes, BaseRecipe, min_length=0) + recipes = {recipe.name:recipe for recipe in recipes} + else: + # Validate the recipe dictionary + valid_dict(recipes, str, BaseRecipe, strict=False, min_length=0) + for k, v in recipes.items(): + if k != v.name: + raise KeyError( + f"Key '{k}' indexes unexpected Recipe '{v.name}' " + "Recipe dictionaries must be keyed with the name of the " + "Recipe.") + + # Try to create a rule for each rule in turn + generated_rules = {} + for pattern in patterns.values(): + if pattern.recipe in recipes: + try: + rule = create_rule(pattern, recipes[pattern.recipe]) + generated_rules[rule.name] = rule + except TypeError: + pass + return generated_rules + +def create_rule(pattern:BasePattern, recipe:BaseRecipe, + new_rules:List[BaseRule]=[])->BaseRule: + """Function to create a valid rule from a given pattern and recipe. All + inbuilt rule types are considered, with additional definitions provided + through the 'new_rules' variable.""" + check_type(pattern, BasePattern, hint="create_rule.pattern") + check_type(recipe, BaseRecipe, hint="create_rule.recipe") + valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules") + + # TODO fix me + # Imported here to avoid circular imports at top of file + import rules + all_rules = {(r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()} + + # Add in new rules + for rule in new_rules: + all_rules[(rule.pattern_type, rule.recipe_type)] = rule + + # Find appropriate rule type from pattern and recipe types + key = (type(pattern).__name__, type(recipe).__name__) + if (key) in all_rules: + return all_rules[key]( + generate_rule_id(), + pattern, + recipe + ) + # Raise error if not valid rule type can be found + raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe " + f"'{recipe}' could be found.") diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 381997e..65f5d39 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -18,17 +18,19 @@ from typing import Any, Union, Dict, List from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler +from core.base_recipe import BaseRecipe +from core.base_monitor import BaseMonitor +from core.base_pattern import BasePattern +from core.base_rule import BaseRule from core.correctness.validation import check_type, valid_string, \ valid_dict, valid_list, valid_path, valid_dir_path from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \ FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \ FILE_RETROACTIVE_EVENT, SHA256 -from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \ - create_rule from functionality.debug import setup_debugging, print_debug from functionality.hashing import get_file_hash -from functionality.meow import create_watchdog_event +from functionality.meow import create_rule, create_watchdog_event # Events that are monitored by default _DEFAULT_MASK = [ diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index d9fb877..e7dc080 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -11,6 +11,8 @@ import sys from typing import Any, Tuple, Dict +from core.base_recipe import BaseRecipe +from core.base_handler import BaseHandler from core.correctness.validation import check_type, valid_string, \ valid_dict, valid_path, valid_dir_path, valid_event from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ @@ -19,7 +21,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ JOB_ID, WATCHDOG_BASE, META_FILE, \ PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \ EVENT_RULE, get_base_file -from core.meow import BaseRecipe, BaseHandler from functionality.debug import setup_debugging, print_debug from functionality.file_io import make_dir, read_notebook, write_notebook, \ write_yaml diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index 639548e..ed8b52b 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -10,6 +10,8 @@ import sys from typing import Any, Tuple, Dict, List +from core.base_recipe import BaseRecipe +from core.base_handler import BaseHandler from core.correctness.validation import check_script, valid_string, \ valid_dict, valid_event, valid_dir_path from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ @@ -18,7 +20,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ JOB_ID, WATCHDOG_BASE, META_FILE, \ PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \ get_base_file -from core.meow import BaseRecipe, BaseHandler from functionality.debug import setup_debugging, print_debug from functionality.file_io import make_dir, read_file_lines, write_file, \ write_yaml, lines_to_string diff --git a/rules/file_event_jupyter_notebook_rule.py b/rules/file_event_jupyter_notebook_rule.py index e45c552..12e6300 100644 --- a/rules/file_event_jupyter_notebook_rule.py +++ b/rules/file_event_jupyter_notebook_rule.py @@ -5,8 +5,9 @@ and JupyterNotebookRecipe. Author(s): David Marchant """ + +from core.base_rule import BaseRule from core.correctness.validation import check_type -from core.meow import BaseRule from patterns.file_event_pattern import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe diff --git a/rules/file_event_python_rule.py b/rules/file_event_python_rule.py index d6f3f36..600de46 100644 --- a/rules/file_event_python_rule.py +++ b/rules/file_event_python_rule.py @@ -5,8 +5,9 @@ and PythonRecipe. Author(s): David Marchant """ + +from core.base_rule import BaseRule from core.correctness.validation import check_type -from core.meow import BaseRule from patterns.file_event_pattern import FileEventPattern from recipes.python_recipe import PythonRecipe diff --git a/tests/shared.py b/tests/shared.py index 10793a2..67054e1 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -7,7 +7,8 @@ import os from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR from functionality.file_io import make_dir, rmtree - +from patterns import FileEventPattern +from recipes import JupyterNotebookRecipe # testing TEST_DIR = "test_files" @@ -15,6 +16,7 @@ TEST_MONITOR_BASE = "test_monitor_base" TEST_JOB_QUEUE = "test_job_queue_dir" TEST_JOB_OUTPUT = "test_job_output" + def setup(): make_dir(TEST_DIR, ensure_clean=True) make_dir(TEST_MONITOR_BASE, ensure_clean=True) @@ -302,3 +304,13 @@ ADDING_NOTEBOOK = { "nbformat": 4, "nbformat_minor": 4 } + +valid_pattern_one = FileEventPattern( + "pattern_one", "path_one", "recipe_one", "file_one") +valid_pattern_two = FileEventPattern( + "pattern_two", "path_two", "recipe_two", "file_two") + +valid_recipe_one = JupyterNotebookRecipe( + "recipe_one", BAREBONES_NOTEBOOK) +valid_recipe_two = JupyterNotebookRecipe( + "recipe_two", BAREBONES_NOTEBOOK) diff --git a/tests/test_conductors.py b/tests/test_conductors.py index 7a90b2b..3e992d0 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -11,12 +11,11 @@ from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \ STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \ JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file -from core.meow import create_rule from conductors import LocalPythonConductor from functionality.file_io import read_file, read_yaml, write_file, \ write_notebook, write_yaml, lines_to_string, make_dir from functionality.hashing import get_file_hash -from functionality.meow import create_watchdog_event, create_job +from functionality.meow import create_watchdog_event, create_job, create_rule from patterns import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ papermill_job_func diff --git a/tests/test_functionality.py b/tests/test_functionality.py index db81654..9b646a0 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -6,20 +6,21 @@ import os from datetime import datetime from multiprocessing import Pipe, Queue from time import sleep +from typing import Dict +from core.base_rule import BaseRule from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \ WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \ PYTHON_FUNC, JOB_ID, JOB_EVENT, \ JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL -from core.meow import create_rule from functionality.file_io import lines_to_string, make_dir, read_file, \ read_file_lines, read_notebook, read_yaml, rmtree, write_file, \ write_notebook, write_yaml from functionality.hashing import get_file_hash -from functionality.meow import create_event, create_job, \ - create_watchdog_event, replace_keywords, \ +from functionality.meow import create_event, create_job, create_rule, \ + create_rules, create_watchdog_event, replace_keywords, \ KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \ KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \ KEYWORD_REL_PATH @@ -29,8 +30,9 @@ from functionality.parameterisation import parameterize_jupyter_notebook, \ from functionality.process_io import wait from patterns import FileEventPattern from recipes import JupyterNotebookRecipe -from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \ - APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT +from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \ + valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \ + COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT class DebugTests(unittest.TestCase): def setUp(self)->None: @@ -553,6 +555,58 @@ class MeowTests(unittest.TestCase): self.assertEqual(replaced["M"], "A") self.assertEqual(replaced["N"], 1) + # Test that create_rule creates a rule from pattern and recipe + def testCreateRule(self)->None: + rule = create_rule(valid_pattern_one, valid_recipe_one) + + self.assertIsInstance(rule, BaseRule) + + with self.assertRaises(ValueError): + rule = create_rule(valid_pattern_one, valid_recipe_two) + + # Test that create_rules creates nothing from nothing + def testCreateRulesMinimum(self)->None: + rules = create_rules({}, {}) + + self.assertEqual(len(rules), 0) + + # Test that create_rules creates rules from patterns and recipes + def testCreateRulesPatternsAndRecipesDicts(self)->None: + patterns = { + valid_pattern_one.name: valid_pattern_one, + valid_pattern_two.name: valid_pattern_two + } + recipes = { + valid_recipe_one.name: valid_recipe_one, + valid_recipe_two.name: valid_recipe_two + } + rules = create_rules(patterns, recipes) + self.assertIsInstance(rules, Dict) + self.assertEqual(len(rules), 2) + for k, rule in rules.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(rule, BaseRule) + self.assertEqual(k, rule.name) + + # Test that create_rules creates nothing from invalid pattern inputs + def testCreateRulesMisindexedPatterns(self)->None: + patterns = { + valid_pattern_two.name: valid_pattern_one, + valid_pattern_one.name: valid_pattern_two + } + with self.assertRaises(KeyError): + create_rules(patterns, {}) + + # Test that create_rules creates nothing from invalid recipe inputs + def testCreateRulesMisindexedRecipes(self)->None: + recipes = { + valid_recipe_two.name: valid_recipe_one, + valid_recipe_one.name: valid_recipe_two + } + with self.assertRaises(KeyError): + create_rules({}, recipes) + + class NamingTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_meow.py b/tests/test_meow.py index 6186065..6ecb1c7 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -3,24 +3,18 @@ import unittest from typing import Any, Union, Tuple, Dict +from core.base_conductor import BaseConductor +from core.base_handler import BaseHandler +from core.base_monitor import BaseMonitor +from core.base_pattern import BasePattern +from core.base_recipe import BaseRecipe +from core.base_rule import BaseRule from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START -from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \ - BaseHandler, BaseConductor, create_rules, create_rule from patterns import FileEventPattern -from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe -from shared import setup, teardown, BAREBONES_NOTEBOOK - -valid_pattern_one = FileEventPattern( - "pattern_one", "path_one", "recipe_one", "file_one") -valid_pattern_two = FileEventPattern( - "pattern_two", "path_two", "recipe_two", "file_two") - -valid_recipe_one = JupyterNotebookRecipe( - "recipe_one", BAREBONES_NOTEBOOK) -valid_recipe_two = JupyterNotebookRecipe( - "recipe_two", BAREBONES_NOTEBOOK) +from shared import setup, teardown, valid_pattern_one, valid_recipe_one +# TODO split me class MeowTests(unittest.TestCase): def setUp(self)->None: super().setUp() @@ -162,57 +156,6 @@ class MeowTests(unittest.TestCase): pass FullRule("name", valid_pattern_one, valid_recipe_one) - # Test that create_rule creates a rule from pattern and recipe - def testCreateRule(self)->None: - rule = create_rule(valid_pattern_one, valid_recipe_one) - - self.assertIsInstance(rule, BaseRule) - - with self.assertRaises(ValueError): - rule = create_rule(valid_pattern_one, valid_recipe_two) - - # Test that create_rules creates nothing from nothing - def testCreateRulesMinimum(self)->None: - rules = create_rules({}, {}) - - self.assertEqual(len(rules), 0) - - # Test that create_rules creates rules from patterns and recipes - def testCreateRulesPatternsAndRecipesDicts(self)->None: - patterns = { - valid_pattern_one.name: valid_pattern_one, - valid_pattern_two.name: valid_pattern_two - } - recipes = { - valid_recipe_one.name: valid_recipe_one, - valid_recipe_two.name: valid_recipe_two - } - rules = create_rules(patterns, recipes) - self.assertIsInstance(rules, Dict) - self.assertEqual(len(rules), 2) - for k, rule in rules.items(): - self.assertIsInstance(k, str) - self.assertIsInstance(rule, BaseRule) - self.assertEqual(k, rule.name) - - # Test that create_rules creates nothing from invalid pattern inputs - def testCreateRulesMisindexedPatterns(self)->None: - patterns = { - valid_pattern_two.name: valid_pattern_one, - valid_pattern_one.name: valid_pattern_two - } - with self.assertRaises(KeyError): - create_rules(patterns, {}) - - # Test that create_rules creates nothing from invalid recipe inputs - def testCreateRulesMisindexedRecipes(self)->None: - recipes = { - valid_recipe_two.name: valid_recipe_one, - valid_recipe_one.name: valid_recipe_two - } - with self.assertRaises(KeyError): - create_rules({}, recipes) - # Test that BaseMonitor instantiation def testBaseMonitor(self)->None: with self.assertRaises(TypeError): diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 5276543..cc05cc0 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -13,11 +13,11 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \ get_base_file, get_job_file, get_result_file from core.correctness.validation import valid_job -from core.meow import create_rules, create_rule from functionality.file_io import lines_to_string, make_dir, read_yaml, \ write_file, write_notebook, write_yaml from functionality.hashing import get_file_hash -from functionality.meow import create_job, create_watchdog_event +from functionality.meow import create_job, create_rules, create_rule, \ + create_watchdog_event from patterns.file_event_pattern import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler, papermill_job_func diff --git a/tests/test_runner.py b/tests/test_runner.py index a65c219..0ca0b38 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -5,10 +5,12 @@ import unittest from time import sleep +from core.base_conductor import BaseConductor +from core.base_handler import BaseHandler +from core.base_monitor import BaseMonitor from conductors import LocalPythonConductor from core.correctness.vars import get_result_file, \ JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME -from core.meow import BaseMonitor, BaseHandler, BaseConductor from core.runner import MeowRunner from functionality.file_io import make_dir, read_file, read_notebook, read_yaml from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern