NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

also refactored core.meow into seperate files in hope that it'll help solve circular imports

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-10 16:23:24 +01:00
parent b8885146fa
commit 6fab2f7a10
22 changed files with 701 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -51,7 +51,7 @@ coverage.xml
.hypothesis/
.pytest_cache/
tests/test_monitor_base
tests/test_job_queue_dir_dir_dir_dir_dir_dir_dir_dir_dir_dir
tests/test_job_queue_dir
tests/test_job_output
# Translations

2
conductors/local_python_conductor.py
View File

@ -11,12 +11,12 @@ import shutil
from datetime import datetime
from typing import Any, Tuple, Dict
from core.base_conductor import BaseConductor
from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
from core.correctness.validation import valid_job, valid_dir_path
from core.meow import BaseConductor
from functionality.file_io import make_dir, read_yaml, write_file, write_yaml
class LocalPythonConductor(BaseConductor):

46
core/base_conductor.py Normal file
View File

@ -0,0 +1,46 @@
"""
This file contains the base MEOW conductor defintion. This should be inherited
from for all conductor instances.
Author(s): David Marchant
"""
from typing import Any, Tuple, Dict
from core.correctness.vars import get_drt_imp_msg
from core.correctness.validation import check_implementation
class BaseConductor:
# Directory where queued jobs are initially written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_queue_dir:str
# Directory where completed jobs are finally written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_output_dir:str
def __init__(self)->None:
"""BaseConductor Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).execute, BaseConductor)
check_implementation(type(self).valid_execute_criteria, BaseConductor)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseConductor:
msg = get_drt_imp_msg(BaseConductor)
raise TypeError(msg)
return object.__new__(cls)
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
process it or not. Must be implemented by any child process."""
pass
def execute(self, job_dir:str)->None:
"""Function to execute a given job directory. Must be implemented by
any child process."""
pass

46
core/base_handler.py Normal file
View File

@ -0,0 +1,46 @@
"""
This file contains the base MEOW handler defintion. This should be inherited
from for all handler instances.
Author(s): David Marchant
"""
from typing import Any, Tuple, Dict
from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from core.correctness.validation import check_implementation
class BaseHandler:
# A channel for sending messages to the runner. Note that this will be
# overridden by a MeowRunner, if a handler instance is passed to it, and so
# does not need to be initialised within the handler itself.
to_runner: VALID_CHANNELS
# Directory where queued jobs are initially written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_queue_dir:str
def __init__(self)->None:
"""BaseHandler Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).handle, BaseHandler)
check_implementation(type(self).valid_handle_criteria, BaseHandler)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseHandler:
msg = get_drt_imp_msg(BaseHandler)
raise TypeError(msg)
return object.__new__(cls)
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. Must be implemented by any child process."""
pass
def handle(self, event:Dict[str,Any])->None:
"""Function to handle a given event. Must be implemented by any child
process."""
pass

128
core/base_monitor.py Normal file
View File

@ -0,0 +1,128 @@
"""
This file contains the base MEOW monitor defintion. This should be inherited
from for all monitor instances.
Author(s): David Marchant
"""
from copy import deepcopy
from typing import Union, Dict
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from core.correctness.validation import check_implementation
from functionality.meow import create_rules
class BaseMonitor:
# A collection of patterns
_patterns: Dict[str, BasePattern]
# A collection of recipes
_recipes: Dict[str, BaseRecipe]
# A collection of rules derived from _patterns and _recipes
_rules: Dict[str, BaseRule]
# A channel for sending messages to the runner. Note that this is not
# initialised within the constructor, but within the runner when passed the
# monitor is passed to it.
to_runner: VALID_CHANNELS
def __init__(self, patterns:Dict[str,BasePattern],
recipes:Dict[str,BaseRecipe])->None:
"""BaseMonitor Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self).start, BaseMonitor)
check_implementation(type(self).stop, BaseMonitor)
check_implementation(type(self)._is_valid_patterns, BaseMonitor)
self._is_valid_patterns(patterns)
check_implementation(type(self)._is_valid_recipes, BaseMonitor)
self._is_valid_recipes(recipes)
check_implementation(type(self).add_pattern, BaseMonitor)
check_implementation(type(self).update_pattern, BaseMonitor)
check_implementation(type(self).remove_pattern, BaseMonitor)
check_implementation(type(self).get_patterns, BaseMonitor)
check_implementation(type(self).add_recipe, BaseMonitor)
check_implementation(type(self).update_recipe, BaseMonitor)
check_implementation(type(self).remove_recipe, BaseMonitor)
check_implementation(type(self).get_recipes, BaseMonitor)
check_implementation(type(self).get_rules, BaseMonitor)
# Ensure that patterns and recipes cannot be trivially modified from
# outside the monitor, as this will cause internal consistency issues
self._patterns = deepcopy(patterns)
self._recipes = deepcopy(recipes)
self._rules = create_rules(patterns, recipes)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseMonitor:
msg = get_drt_imp_msg(BaseMonitor)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
"""Validation check for 'patterns' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
"""Validation check for 'recipes' variable from main constructor. Must
be implemented by any child class."""
pass
def start(self)->None:
"""Function to start the monitor as an ongoing process/thread. Must be
implemented by any child process"""
pass
def stop(self)->None:
"""Function to stop the monitor as an ongoing process/thread. Must be
implemented by any child process"""
pass
def add_pattern(self, pattern:BasePattern)->None:
"""Function to add a pattern to the current definitions. Must be
implemented by any child process."""
pass
def update_pattern(self, pattern:BasePattern)->None:
"""Function to update a pattern in the current definitions. Must be
implemented by any child process."""
pass
def remove_pattern(self, pattern:Union[str,BasePattern])->None:
"""Function to remove a pattern from the current definitions. Must be
implemented by any child process."""
pass
def get_patterns(self)->Dict[str,BasePattern]:
"""Function to get a dictionary of all current pattern definitions.
Must be implemented by any child process."""
pass
def add_recipe(self, recipe:BaseRecipe)->None:
"""Function to add a recipe to the current definitions. Must be
implemented by any child process."""
pass
def update_recipe(self, recipe:BaseRecipe)->None:
"""Function to update a recipe in the current definitions. Must be
implemented by any child process."""
pass
def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
"""Function to remove a recipe from the current definitions. Must be
implemented by any child process."""
pass
def get_recipes(self)->Dict[str,BaseRecipe]:
"""Function to get a dictionary of all current recipe definitions.
Must be implemented by any child process."""
pass
def get_rules(self)->Dict[str,BaseRule]:
"""Function to get a dictionary of all current rule definitions.
Must be implemented by any child process."""
pass

141
core/base_pattern.py Normal file
View File

@ -0,0 +1,141 @@
"""
This file contains the base MEOW pattern defintion. This should be inherited
from for all pattern instances.
Author(s): David Marchant
"""
import itertools
from typing import Any, Union, Tuple, Dict, List
from core.correctness.vars import get_drt_imp_msg, \
VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
from core.correctness.validation import valid_string, check_type, \
check_implementation, valid_dict
class BasePattern:
# A unique identifier for the pattern
name:str
# An identifier of a recipe
recipe:str
# Parameters to be overridden in the recipe
parameters:Dict[str,Any]
# Parameters showing the potential outputs of a recipe
outputs:Dict[str,Any]
# A collection of variables to be swept over for job scheduling
sweep:Dict[str,Any]
def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={},
outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}):
"""BasePattern Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_recipe, BasePattern)
check_implementation(type(self)._is_valid_parameters, BasePattern)
check_implementation(type(self)._is_valid_output, BasePattern)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
self.recipe = recipe
self._is_valid_parameters(parameters)
self.parameters = parameters
self._is_valid_output(outputs)
self.outputs = outputs
self._is_valid_sweep(sweep)
self.sweep = sweep
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BasePattern:
msg = get_drt_imp_msg(BasePattern)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_PATTERN_NAME_CHARS)
def _is_valid_recipe(self, recipe:Any)->None:
"""Validation check for 'recipe' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_parameters(self, parameters:Any)->None:
"""Validation check for 'parameters' variable from main constructor.
Must be implemented by any child class."""
pass
def _is_valid_output(self, outputs:Any)->None:
"""Validation check for 'outputs' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None:
"""Validation check for 'sweep' variable from main constructor. This
function is implemented to check for the types given in the signature,
and must be overridden if these differ."""
check_type(sweep, Dict, hint="BasePattern.sweep")
if not sweep:
return
for _, v in sweep.items():
valid_dict(
v, str, Any, [
SWEEP_START, SWEEP_STOP, SWEEP_JUMP
], strict=True)
check_type(
v[SWEEP_START],
expected_type=int,
alt_types=[float, complex],
hint=f"BasePattern.sweep[{SWEEP_START}]"
)
check_type(
v[SWEEP_STOP],
expected_type=int,
alt_types=[float, complex],
hint=f"BasePattern.sweep[{SWEEP_STOP}]"
)
check_type(
v[SWEEP_JUMP],
expected_type=int,
alt_types=[float, complex],
hint=f"BasePattern.sweep[{SWEEP_JUMP}]"
)
# Try to check that this loop is not infinite
if v[SWEEP_JUMP] == 0:
raise ValueError(
f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
)
elif v[SWEEP_JUMP] > 0:
if not v[SWEEP_STOP] > v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
elif v[SWEEP_JUMP] < 0:
if not v[SWEEP_STOP] < v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
def expand_sweeps(self)->List[Tuple[str,Any]]:
"""Function to get all combinations of sweep parameters"""
values_dict = {}
# get a collection of a individual sweep values
for var, val in self.sweep.items():
values_dict[var] = []
par_val = val[SWEEP_START]
while par_val <= val[SWEEP_STOP]:
values_dict[var].append((var, par_val))
par_val += val[SWEEP_JUMP]
# combine all combinations of sweep values
return list(itertools.product(
*[v for v in values_dict.values()]))

68
core/base_recipe.py Normal file
View File

@ -0,0 +1,68 @@
"""
This file contains the base MEOW recipe defintion. This should be inherited
from for all recipe instances.
Author(s): David Marchant
"""
from typing import Any, Dict
from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
from core.correctness.validation import valid_string, check_implementation
class BaseRecipe:
# A unique identifier for the recipe
name:str
# Actual code to run
recipe:Any
# Possible parameters that could be overridden by a Pattern
parameters:Dict[str, Any]
# Additional configuration options
requirements:Dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={},
requirements:Dict[str,Any]={}):
"""BaseRecipe Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_recipe, BaseRecipe)
check_implementation(type(self)._is_valid_parameters, BaseRecipe)
check_implementation(type(self)._is_valid_requirements, BaseRecipe)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
self.recipe = recipe
self._is_valid_parameters(parameters)
self.parameters = parameters
self._is_valid_requirements(requirements)
self.requirements = requirements
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseRecipe:
msg = get_drt_imp_msg(BaseRecipe)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_RECIPE_NAME_CHARS)
def _is_valid_recipe(self, recipe:Any)->None:
"""Validation check for 'recipe' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_parameters(self, parameters:Any)->None:
"""Validation check for 'parameters' variable from main constructor.
Must be implemented by any child class."""
pass
def _is_valid_requirements(self, requirements:Any)->None:
"""Validation check for 'requirements' variable from main constructor.
Must be implemented by any child class."""
pass

81
core/base_rule.py Normal file
View File

@ -0,0 +1,81 @@
"""
This file contains the base MEOW rule defintion. This should be inherited from
for all rule instances.
Author(s): David Marchant
"""
from typing import Any
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
from core.correctness.validation import valid_string, check_type, \
check_implementation
class BaseRule:
# A unique identifier for the rule
name:str
# A pattern to be used in rule triggering
pattern:BasePattern
# A recipe to be used in rule execution
recipe:BaseRecipe
# The string name of the pattern class that can be used to create this rule
pattern_type:str=""
# The string name of the recipe class that can be used to create this rule
recipe_type:str=""
def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
"""BaseRule Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_pattern, BaseRule)
check_implementation(type(self)._is_valid_recipe, BaseRule)
self.__check_types_set()
self._is_valid_name(name)
self.name = name
self._is_valid_pattern(pattern)
self.pattern = pattern
self._is_valid_recipe(recipe)
self.recipe = recipe
check_type(pattern, BasePattern, hint="BaseRule.pattern")
check_type(recipe, BaseRecipe, hint="BaseRule.recipe")
if pattern.recipe != recipe.name:
raise ValueError(f"Cannot create Rule {name}. Pattern "
f"{pattern.name} does not identify Recipe {recipe.name}. It "
f"uses {pattern.recipe}")
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseRule:
msg = get_drt_imp_msg(BaseRule)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_RULE_NAME_CHARS)
def _is_valid_pattern(self, pattern:Any)->None:
"""Validation check for 'pattern' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_recipe(self, recipe:Any)->None:
"""Validation check for 'recipe' variable from main constructor. Must
be implemented by any child class."""
pass
def __check_types_set(self)->None:
"""Validation check that the self.pattern_type and self.recipe_type
attributes have been set in a child class."""
if self.pattern_type == "":
raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
"does not set a pattern_type.")
if self.recipe_type == "":
raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
"does not set a recipe_type.")

541
core/meow.py
View File

@ -1,541 +0,0 @@
"""
This file contains the core MEOW defintions, used throughout this package.
It is intended that these base definitions are what should be inherited from in
order to create an extendable framework for event-based scheduling and
processing.
Author(s): David Marchant
"""
import inspect
import itertools
import sys
from copy import deepcopy
from typing import Any, Union, Tuple, Dict, List
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
from core.correctness.validation import valid_string, check_type, \
check_implementation, valid_list, valid_dict
from functionality.naming import generate_rule_id
class BaseRecipe:
# A unique identifier for the recipe
name:str
# Actual code to run
recipe:Any
# Possible parameters that could be overridden by a Pattern
parameters:Dict[str, Any]
# Additional configuration options
requirements:Dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={},
requirements:Dict[str,Any]={}):
"""BaseRecipe Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_recipe, BaseRecipe)
check_implementation(type(self)._is_valid_parameters, BaseRecipe)
check_implementation(type(self)._is_valid_requirements, BaseRecipe)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
self.recipe = recipe
self._is_valid_parameters(parameters)
self.parameters = parameters
self._is_valid_requirements(requirements)
self.requirements = requirements
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseRecipe:
msg = get_drt_imp_msg(BaseRecipe)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_RECIPE_NAME_CHARS)
def _is_valid_recipe(self, recipe:Any)->None:
"""Validation check for 'recipe' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_parameters(self, parameters:Any)->None:
"""Validation check for 'parameters' variable from main constructor.
Must be implemented by any child class."""
pass
def _is_valid_requirements(self, requirements:Any)->None:
"""Validation check for 'requirements' variable from main constructor.
Must be implemented by any child class."""
pass
class BasePattern:
# A unique identifier for the pattern
name:str
# An identifier of a recipe
recipe:str
# Parameters to be overridden in the recipe
parameters:Dict[str,Any]
# Parameters showing the potential outputs of a recipe
outputs:Dict[str,Any]
# A collection of variables to be swept over for job scheduling
sweep:Dict[str,Any]
def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={},
outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}):
"""BasePattern Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_recipe, BasePattern)
check_implementation(type(self)._is_valid_parameters, BasePattern)
check_implementation(type(self)._is_valid_output, BasePattern)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
self.recipe = recipe
self._is_valid_parameters(parameters)
self.parameters = parameters
self._is_valid_output(outputs)
self.outputs = outputs
self._is_valid_sweep(sweep)
self.sweep = sweep
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BasePattern:
msg = get_drt_imp_msg(BasePattern)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_PATTERN_NAME_CHARS)
def _is_valid_recipe(self, recipe:Any)->None:
"""Validation check for 'recipe' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_parameters(self, parameters:Any)->None:
"""Validation check for 'parameters' variable from main constructor.
Must be implemented by any child class."""
pass
def _is_valid_output(self, outputs:Any)->None:
"""Validation check for 'outputs' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None:
"""Validation check for 'sweep' variable from main constructor. This
function is implemented to check for the types given in the signature,
and must be overridden if these differ."""
check_type(sweep, Dict, hint="BasePattern.sweep")
if not sweep:
return
for _, v in sweep.items():
valid_dict(
v, str, Any, [
SWEEP_START, SWEEP_STOP, SWEEP_JUMP
], strict=True)
check_type(
v[SWEEP_START],
expected_type=int,
alt_types=[float, complex],
hint=f"BasePattern.sweep[{SWEEP_START}]"
)
check_type(
v[SWEEP_STOP],
expected_type=int,
alt_types=[float, complex],
hint=f"BasePattern.sweep[{SWEEP_STOP}]"
)
check_type(
v[SWEEP_JUMP],
expected_type=int,
alt_types=[float, complex],
hint=f"BasePattern.sweep[{SWEEP_JUMP}]"
)
# Try to check that this loop is not infinite
if v[SWEEP_JUMP] == 0:
raise ValueError(
f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
)
elif v[SWEEP_JUMP] > 0:
if not v[SWEEP_STOP] > v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
elif v[SWEEP_JUMP] < 0:
if not v[SWEEP_STOP] < v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
def expand_sweeps(self)->List[Tuple[str,Any]]:
"""Function to get all combinations of sweep parameters"""
values_dict = {}
# get a collection of a individual sweep values
for var, val in self.sweep.items():
values_dict[var] = []
par_val = val[SWEEP_START]
while par_val <= val[SWEEP_STOP]:
values_dict[var].append((var, par_val))
par_val += val[SWEEP_JUMP]
# combine all combinations of sweep values
return list(itertools.product(
*[v for v in values_dict.values()]))
class BaseRule:
# A unique identifier for the rule
name:str
# A pattern to be used in rule triggering
pattern:BasePattern
# A recipe to be used in rule execution
recipe:BaseRecipe
# The string name of the pattern class that can be used to create this rule
pattern_type:str=""
# The string name of the recipe class that can be used to create this rule
recipe_type:str=""
def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
"""BaseRule Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_pattern, BaseRule)
check_implementation(type(self)._is_valid_recipe, BaseRule)
self.__check_types_set()
self._is_valid_name(name)
self.name = name
self._is_valid_pattern(pattern)
self.pattern = pattern
self._is_valid_recipe(recipe)
self.recipe = recipe
check_type(pattern, BasePattern, hint="BaseRule.pattern")
check_type(recipe, BaseRecipe, hint="BaseRule.recipe")
if pattern.recipe != recipe.name:
raise ValueError(f"Cannot create Rule {name}. Pattern "
f"{pattern.name} does not identify Recipe {recipe.name}. It "
f"uses {pattern.recipe}")
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseRule:
msg = get_drt_imp_msg(BaseRule)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_RULE_NAME_CHARS)
def _is_valid_pattern(self, pattern:Any)->None:
"""Validation check for 'pattern' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_recipe(self, recipe:Any)->None:
"""Validation check for 'recipe' variable from main constructor. Must
be implemented by any child class."""
pass
def __check_types_set(self)->None:
"""Validation check that the self.pattern_type and self.recipe_type
attributes have been set in a child class."""
if self.pattern_type == "":
raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
"does not set a pattern_type.")
if self.recipe_type == "":
raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
"does not set a recipe_type.")
class BaseMonitor:
# A collection of patterns
_patterns: Dict[str, BasePattern]
# A collection of recipes
_recipes: Dict[str, BaseRecipe]
# A collection of rules derived from _patterns and _recipes
_rules: Dict[str, BaseRule]
# A channel for sending messages to the runner. Note that this is not
# initialised within the constructor, but within the runner when passed the
# monitor is passed to it.
to_runner: VALID_CHANNELS
def __init__(self, patterns:Dict[str,BasePattern],
recipes:Dict[str,BaseRecipe])->None:
"""BaseMonitor Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self).start, BaseMonitor)
check_implementation(type(self).stop, BaseMonitor)
check_implementation(type(self)._is_valid_patterns, BaseMonitor)
self._is_valid_patterns(patterns)
check_implementation(type(self)._is_valid_recipes, BaseMonitor)
self._is_valid_recipes(recipes)
check_implementation(type(self).add_pattern, BaseMonitor)
check_implementation(type(self).update_pattern, BaseMonitor)
check_implementation(type(self).remove_pattern, BaseMonitor)
check_implementation(type(self).get_patterns, BaseMonitor)
check_implementation(type(self).add_recipe, BaseMonitor)
check_implementation(type(self).update_recipe, BaseMonitor)
check_implementation(type(self).remove_recipe, BaseMonitor)
check_implementation(type(self).get_recipes, BaseMonitor)
check_implementation(type(self).get_rules, BaseMonitor)
# Ensure that patterns and recipes cannot be trivially modified from
# outside the monitor, as this will cause internal consistency issues
self._patterns = deepcopy(patterns)
self._recipes = deepcopy(recipes)
self._rules = create_rules(patterns, recipes)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseMonitor:
msg = get_drt_imp_msg(BaseMonitor)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
"""Validation check for 'patterns' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
"""Validation check for 'recipes' variable from main constructor. Must
be implemented by any child class."""
pass
def start(self)->None:
"""Function to start the monitor as an ongoing process/thread. Must be
implemented by any child process"""
pass
def stop(self)->None:
"""Function to stop the monitor as an ongoing process/thread. Must be
implemented by any child process"""
pass
def add_pattern(self, pattern:BasePattern)->None:
"""Function to add a pattern to the current definitions. Must be
implemented by any child process."""
pass
def update_pattern(self, pattern:BasePattern)->None:
"""Function to update a pattern in the current definitions. Must be
implemented by any child process."""
pass
def remove_pattern(self, pattern:Union[str,BasePattern])->None:
"""Function to remove a pattern from the current definitions. Must be
implemented by any child process."""
pass
def get_patterns(self)->Dict[str,BasePattern]:
"""Function to get a dictionary of all current pattern definitions.
Must be implemented by any child process."""
pass
def add_recipe(self, recipe:BaseRecipe)->None:
"""Function to add a recipe to the current definitions. Must be
implemented by any child process."""
pass
def update_recipe(self, recipe:BaseRecipe)->None:
"""Function to update a recipe in the current definitions. Must be
implemented by any child process."""
pass
def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
"""Function to remove a recipe from the current definitions. Must be
implemented by any child process."""
pass
def get_recipes(self)->Dict[str,BaseRecipe]:
"""Function to get a dictionary of all current recipe definitions.
Must be implemented by any child process."""
pass
def get_rules(self)->Dict[str,BaseRule]:
"""Function to get a dictionary of all current rule definitions.
Must be implemented by any child process."""
pass
class BaseHandler:
# A channel for sending messages to the runner. Note that this will be
# overridden by a MeowRunner, if a handler instance is passed to it, and so
# does not need to be initialised within the handler itself.
to_runner: VALID_CHANNELS
# Directory where queued jobs are initially written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_queue_dir:str
def __init__(self)->None:
"""BaseHandler Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).handle, BaseHandler)
check_implementation(type(self).valid_handle_criteria, BaseHandler)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseHandler:
msg = get_drt_imp_msg(BaseHandler)
raise TypeError(msg)
return object.__new__(cls)
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. Must be implemented by any child process."""
pass
def handle(self, event:Dict[str,Any])->None:
"""Function to handle a given event. Must be implemented by any child
process."""
pass
class BaseConductor:
# Directory where queued jobs are initially written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_queue_dir:str
# Directory where completed jobs are finally written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_output_dir:str
def __init__(self)->None:
"""BaseConductor Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).execute, BaseConductor)
check_implementation(type(self).valid_execute_criteria, BaseConductor)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseConductor:
msg = get_drt_imp_msg(BaseConductor)
raise TypeError(msg)
return object.__new__(cls)
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
process it or not. Must be implemented by any child process."""
pass
def execute(self, job_dir:str)->None:
"""Function to execute a given job directory. Must be implemented by
any child process."""
pass
def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
"""Function to create any valid rules from a given collection of patterns
and recipes. All inbuilt rule types are considered, with additional
definitions provided through the 'new_rules' variable. Note that any
provided pattern and recipe dictionaries must be keyed with the
corresponding pattern and recipe names."""
# Validation of inputs
check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns")
check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes")
valid_list(new_rules, BaseRule, min_length=0)
# Convert a pattern list to a dictionary
if isinstance(patterns, list):
valid_list(patterns, BasePattern, min_length=0)
patterns = {pattern.name:pattern for pattern in patterns}
else:
# Validate the pattern dictionary
valid_dict(patterns, str, BasePattern, strict=False, min_length=0)
for k, v in patterns.items():
if k != v.name:
raise KeyError(
f"Key '{k}' indexes unexpected Pattern '{v.name}' "
"Pattern dictionaries must be keyed with the name of the "
"Pattern.")
# Convert a recipe list into a dictionary
if isinstance(recipes, list):
valid_list(recipes, BaseRecipe, min_length=0)
recipes = {recipe.name:recipe for recipe in recipes}
else:
# Validate the recipe dictionary
valid_dict(recipes, str, BaseRecipe, strict=False, min_length=0)
for k, v in recipes.items():
if k != v.name:
raise KeyError(
f"Key '{k}' indexes unexpected Recipe '{v.name}' "
"Recipe dictionaries must be keyed with the name of the "
"Recipe.")
# Try to create a rule for each rule in turn
generated_rules = {}
for pattern in patterns.values():
if pattern.recipe in recipes:
try:
rule = create_rule(pattern, recipes[pattern.recipe])
generated_rules[rule.name] = rule
except TypeError:
pass
return generated_rules
def create_rule(pattern:BasePattern, recipe:BaseRecipe,
new_rules:List[BaseRule]=[])->BaseRule:
"""Function to create a valid rule from a given pattern and recipe. All
inbuilt rule types are considered, with additional definitions provided
through the 'new_rules' variable."""
check_type(pattern, BasePattern, hint="create_rule.pattern")
check_type(recipe, BaseRecipe, hint="create_rule.recipe")
valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules")
print("passed initial check")
# Imported here to avoid circular imports at top of file
import rules
# Get a dictionary of all inbuilt rules
all_rules ={(r.pattern_type, r.recipe_type):r for r in [r[1] \
for r in inspect.getmembers(sys.modules["rules"], inspect.isclass) \
if (issubclass(r[1], BaseRule))]}
print("got base rules")
# Add in new rules
for rule in new_rules:
all_rules[(rule.pattern_type, rule.recipe_type)] = rule
print("got new rules")
# Find appropriate rule type from pattern and recipe types
key = (type(pattern).__name__, type(recipe).__name__)
print("got key")
if (key) in all_rules:
return all_rules[key](
generate_rule_id(),
pattern,
recipe
)
print("no key")
# Raise error if not valid rule type can be found
raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe "
f"'{recipe}' could be found.")

4
core/runner.py
View File

@ -14,10 +14,12 @@ from multiprocessing import Pipe
from random import randrange
from typing import Any, Union, Dict, List
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from core.correctness.validation import check_type, valid_list, valid_dir_path
from core.meow import BaseHandler, BaseMonitor, BaseConductor
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_yaml
from functionality.process_io import wait

89
functionality/meow.py
View File

@ -6,13 +6,17 @@ Author(s): David Marchant
from datetime import datetime
from os.path import basename, dirname, relpath, splitext
from typing import Any, Dict
from typing import Any, Dict, Union, List
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.validation import check_type, valid_dict, valid_list
from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \
STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH
from functionality.naming import generate_job_id
from functionality.naming import generate_job_id, generate_rule_id
# mig trigger keyword replacements
@ -101,3 +105,84 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
return {**extras, **job_dict}
def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
"""Function to create any valid rules from a given collection of patterns
and recipes. All inbuilt rule types are considered, with additional
definitions provided through the 'new_rules' variable. Note that any
provided pattern and recipe dictionaries must be keyed with the
corresponding pattern and recipe names."""
# Validation of inputs
check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns")
check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes")
valid_list(new_rules, BaseRule, min_length=0)
# Convert a pattern list to a dictionary
if isinstance(patterns, list):
valid_list(patterns, BasePattern, min_length=0)
patterns = {pattern.name:pattern for pattern in patterns}
else:
# Validate the pattern dictionary
valid_dict(patterns, str, BasePattern, strict=False, min_length=0)
for k, v in patterns.items():
if k != v.name:
raise KeyError(
f"Key '{k}' indexes unexpected Pattern '{v.name}' "
"Pattern dictionaries must be keyed with the name of the "
"Pattern.")
# Convert a recipe list into a dictionary
if isinstance(recipes, list):
valid_list(recipes, BaseRecipe, min_length=0)
recipes = {recipe.name:recipe for recipe in recipes}
else:
# Validate the recipe dictionary
valid_dict(recipes, str, BaseRecipe, strict=False, min_length=0)
for k, v in recipes.items():
if k != v.name:
raise KeyError(
f"Key '{k}' indexes unexpected Recipe '{v.name}' "
"Recipe dictionaries must be keyed with the name of the "
"Recipe.")
# Try to create a rule for each rule in turn
generated_rules = {}
for pattern in patterns.values():
if pattern.recipe in recipes:
try:
rule = create_rule(pattern, recipes[pattern.recipe])
generated_rules[rule.name] = rule
except TypeError:
pass
return generated_rules
def create_rule(pattern:BasePattern, recipe:BaseRecipe,
new_rules:List[BaseRule]=[])->BaseRule:
"""Function to create a valid rule from a given pattern and recipe. All
inbuilt rule types are considered, with additional definitions provided
through the 'new_rules' variable."""
check_type(pattern, BasePattern, hint="create_rule.pattern")
check_type(recipe, BaseRecipe, hint="create_rule.recipe")
valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules")
# TODO fix me
# Imported here to avoid circular imports at top of file
import rules
all_rules = {(r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()}
# Add in new rules
for rule in new_rules:
all_rules[(rule.pattern_type, rule.recipe_type)] = rule
# Find appropriate rule type from pattern and recipe types
key = (type(pattern).__name__, type(recipe).__name__)
if (key) in all_rules:
return all_rules[key](
generate_rule_id(),
pattern,
recipe
)
# Raise error if not valid rule type can be found
raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe "
f"'{recipe}' could be found.")

8
patterns/file_event_pattern.py
View File

@ -18,17 +18,19 @@ from typing import Any, Union, Dict, List
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from core.base_recipe import BaseRecipe
from core.base_monitor import BaseMonitor
from core.base_pattern import BasePattern
from core.base_rule import BaseRule
from core.correctness.validation import check_type, valid_string, \
valid_dict, valid_list, valid_path, valid_dir_path
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
FILE_RETROACTIVE_EVENT, SHA256
from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \
create_rule
from functionality.debug import setup_debugging, print_debug
from functionality.hashing import get_file_hash
from functionality.meow import create_watchdog_event
from functionality.meow import create_rule, create_watchdog_event
# Events that are monitored by default
_DEFAULT_MASK = [

3
recipes/jupyter_notebook_recipe.py
View File

@ -11,6 +11,8 @@ import sys
from typing import Any, Tuple, Dict
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path, valid_event
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
@ -19,7 +21,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
EVENT_RULE, get_base_file
from core.meow import BaseRecipe, BaseHandler
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_notebook, write_notebook, \
write_yaml

3
recipes/python_recipe.py
View File

@ -10,6 +10,8 @@ import sys
from typing import Any, Tuple, Dict, List
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.validation import check_script, valid_string, \
valid_dict, valid_event, valid_dir_path
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
@ -18,7 +20,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
get_base_file
from core.meow import BaseRecipe, BaseHandler
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_file_lines, write_file, \
write_yaml, lines_to_string

3
rules/file_event_jupyter_notebook_rule.py
View File

@ -5,8 +5,9 @@ and JupyterNotebookRecipe.
Author(s): David Marchant
"""
from core.base_rule import BaseRule
from core.correctness.validation import check_type
from core.meow import BaseRule
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe

3
rules/file_event_python_rule.py
View File

@ -5,8 +5,9 @@ and PythonRecipe.
Author(s): David Marchant
"""
from core.base_rule import BaseRule
from core.correctness.validation import check_type
from core.meow import BaseRule
from patterns.file_event_pattern import FileEventPattern
from recipes.python_recipe import PythonRecipe

14
tests/shared.py
View File

@ -7,7 +7,8 @@ import os
from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from functionality.file_io import make_dir, rmtree
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
# testing
TEST_DIR = "test_files"
@ -15,6 +16,7 @@ TEST_MONITOR_BASE = "test_monitor_base"
TEST_JOB_QUEUE = "test_job_queue_dir"
TEST_JOB_OUTPUT = "test_job_output"
def setup():
make_dir(TEST_DIR, ensure_clean=True)
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
@ -302,3 +304,13 @@ ADDING_NOTEBOOK = {
"nbformat": 4,
"nbformat_minor": 4
}
valid_pattern_one = FileEventPattern(
"pattern_one", "path_one", "recipe_one", "file_one")
valid_pattern_two = FileEventPattern(
"pattern_two", "path_two", "recipe_two", "file_two")
valid_recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
valid_recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)

3
tests/test_conductors.py
View File

@ -11,12 +11,11 @@ from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
from core.meow import create_rule
from conductors import LocalPythonConductor
from functionality.file_io import read_file, read_yaml, write_file, \
write_notebook, write_yaml, lines_to_string, make_dir
from functionality.hashing import get_file_hash
from functionality.meow import create_watchdog_event, create_job
from functionality.meow import create_watchdog_event, create_job, create_rule
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func

64
tests/test_functionality.py
View File

@ -6,20 +6,21 @@ import os
from datetime import datetime
from multiprocessing import Pipe, Queue
from time import sleep
from typing import Dict
from core.base_rule import BaseRule
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from core.meow import create_rule
from functionality.file_io import lines_to_string, make_dir, read_file, \
read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_event, create_job, \
create_watchdog_event, replace_keywords, \
from functionality.meow import create_event, create_job, create_rule, \
create_rules, create_watchdog_event, replace_keywords, \
KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
KEYWORD_REL_PATH
@ -29,8 +30,9 @@ from functionality.parameterisation import parameterize_jupyter_notebook, \
from functionality.process_io import wait
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
class DebugTests(unittest.TestCase):
def setUp(self)->None:
@ -553,6 +555,58 @@ class MeowTests(unittest.TestCase):
self.assertEqual(replaced["M"], "A")
self.assertEqual(replaced["N"], 1)
# Test that create_rule creates a rule from pattern and recipe
def testCreateRule(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
self.assertIsInstance(rule, BaseRule)
with self.assertRaises(ValueError):
rule = create_rule(valid_pattern_one, valid_recipe_two)
# Test that create_rules creates nothing from nothing
def testCreateRulesMinimum(self)->None:
rules = create_rules({}, {})
self.assertEqual(len(rules), 0)
# Test that create_rules creates rules from patterns and recipes
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,
valid_pattern_two.name: valid_pattern_two
}
recipes = {
valid_recipe_one.name: valid_recipe_one,
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, Dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
self.assertIsInstance(rule, BaseRule)
self.assertEqual(k, rule.name)
# Test that create_rules creates nothing from invalid pattern inputs
def testCreateRulesMisindexedPatterns(self)->None:
patterns = {
valid_pattern_two.name: valid_pattern_one,
valid_pattern_one.name: valid_pattern_two
}
with self.assertRaises(KeyError):
create_rules(patterns, {})
# Test that create_rules creates nothing from invalid recipe inputs
def testCreateRulesMisindexedRecipes(self)->None:
recipes = {
valid_recipe_two.name: valid_recipe_one,
valid_recipe_one.name: valid_recipe_two
}
with self.assertRaises(KeyError):
create_rules({}, recipes)
class NamingTests(unittest.TestCase):
def setUp(self)->None:

73
tests/test_meow.py
View File

@ -3,24 +3,18 @@ import unittest
from typing import Any, Union, Tuple, Dict
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, BaseConductor, create_rules, create_rule
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, BAREBONES_NOTEBOOK
valid_pattern_one = FileEventPattern(
"pattern_one", "path_one", "recipe_one", "file_one")
valid_pattern_two = FileEventPattern(
"pattern_two", "path_two", "recipe_two", "file_two")
valid_recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
valid_recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
from shared import setup, teardown, valid_pattern_one, valid_recipe_one
# TODO split me
class MeowTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -162,57 +156,6 @@ class MeowTests(unittest.TestCase):
pass
FullRule("name", valid_pattern_one, valid_recipe_one)
# Test that create_rule creates a rule from pattern and recipe
def testCreateRule(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
self.assertIsInstance(rule, BaseRule)
with self.assertRaises(ValueError):
rule = create_rule(valid_pattern_one, valid_recipe_two)
# Test that create_rules creates nothing from nothing
def testCreateRulesMinimum(self)->None:
rules = create_rules({}, {})
self.assertEqual(len(rules), 0)
# Test that create_rules creates rules from patterns and recipes
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,
valid_pattern_two.name: valid_pattern_two
}
recipes = {
valid_recipe_one.name: valid_recipe_one,
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, Dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
self.assertIsInstance(rule, BaseRule)
self.assertEqual(k, rule.name)
# Test that create_rules creates nothing from invalid pattern inputs
def testCreateRulesMisindexedPatterns(self)->None:
patterns = {
valid_pattern_two.name: valid_pattern_one,
valid_pattern_one.name: valid_pattern_two
}
with self.assertRaises(KeyError):
create_rules(patterns, {})
# Test that create_rules creates nothing from invalid recipe inputs
def testCreateRulesMisindexedRecipes(self)->None:
recipes = {
valid_recipe_two.name: valid_recipe_one,
valid_recipe_one.name: valid_recipe_two
}
with self.assertRaises(KeyError):
create_rules({}, recipes)
# Test that BaseMonitor instantiation
def testBaseMonitor(self)->None:
with self.assertRaises(TypeError):

4
tests/test_recipes.py
View File

@ -13,11 +13,11 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from core.correctness.validation import valid_job
from core.meow import create_rules, create_rule
from functionality.file_io import lines_to_string, make_dir, read_yaml, \
write_file, write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_job, create_watchdog_event
from functionality.meow import create_job, create_rules, create_rule, \
create_watchdog_event
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, papermill_job_func

4
tests/test_runner.py
View File

@ -5,10 +5,12 @@ import unittest
from time import sleep
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from conductors import LocalPythonConductor
from core.correctness.vars import get_result_file, \
JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME
from core.meow import BaseMonitor, BaseHandler, BaseConductor
from core.runner import MeowRunner
from functionality.file_io import make_dir, read_file, read_notebook, read_yaml
from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern