diff --git a/core/correctness/vars.py b/core/correctness/vars.py index 61a0fd2..103fc4d 100644 --- a/core/correctness/vars.py +++ b/core/correctness/vars.py @@ -113,6 +113,11 @@ PARAMS_FILE = "params.yml" JOB_FILE = "job.ipynb" RESULT_FILE = "result.ipynb" +# Parameter sweep keys +SWEEP_START = "start" +SWEEP_STOP = "stop" +SWEEP_JUMP = "jump" + # debug printing levels DEBUG_ERROR = 1 DEBUG_WARNING = 2 diff --git a/core/meow.py b/core/meow.py index 498948b..fd38b32 100644 --- a/core/meow.py +++ b/core/meow.py @@ -8,6 +8,7 @@ processing. Author(s): David Marchant """ import inspect +import itertools import sys from copy import deepcopy @@ -15,7 +16,7 @@ from typing import Any, Union, Tuple from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \ - get_drt_imp_msg + SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg from core.correctness.validation import valid_string, check_type, \ check_implementation, valid_list, valid_dict from core.functionality import generate_id @@ -86,14 +87,18 @@ class BasePattern: parameters:dict[str,Any] # Parameters showing the potential outputs of a recipe outputs:dict[str,Any] + # A collection of variables to be swept over for job scheduling + sweep:dict[str,Any] + def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={}, - outputs:dict[str,Any]={}): + outputs:dict[str,Any]={}, sweep:dict[str,Any]={}): """BasePattern Constructor. This will check that any class inheriting from it implements its validation functions. It will then call these on the input parameters.""" check_implementation(type(self)._is_valid_recipe, BasePattern) check_implementation(type(self)._is_valid_parameters, BasePattern) check_implementation(type(self)._is_valid_output, BasePattern) + check_implementation(type(self)._is_valid_sweep, BasePattern) self._is_valid_name(name) self.name = name self._is_valid_recipe(recipe) @@ -102,6 +107,8 @@ class BasePattern: self.parameters = parameters self._is_valid_output(outputs) self.outputs = outputs + self._is_valid_sweep(sweep) + self.sweep = sweep def __new__(cls, *args, **kwargs): """A check that this base class is not instantiated itself, only @@ -132,6 +139,57 @@ class BasePattern: be implemented by any child class.""" pass + def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None: + """Validation check for 'sweep' variable from main constructor. Must + be implemented by any child class.""" + check_type(sweep, dict) + if not sweep: + return + for _, v in sweep.items(): + valid_dict( + v, str, Any, [ + SWEEP_START, SWEEP_STOP, SWEEP_JUMP + ], strict=True) + + check_type( + v[SWEEP_START], expected_type=int, alt_types=[float, complex]) + check_type( + v[SWEEP_STOP], expected_type=int, alt_types=[float, complex]) + check_type( + v[SWEEP_JUMP], expected_type=int, alt_types=[float, complex]) + # Try to check that this loop is not infinite + if v[SWEEP_JUMP] == 0: + raise ValueError( + f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero" + ) + elif v[SWEEP_JUMP] > 0: + if not v[SWEEP_STOP] > v[SWEEP_START]: + raise ValueError( + f"Cannot create sweep with a positive '{SWEEP_JUMP}' " + "value where the end point is smaller than the start." + ) + elif v[SWEEP_JUMP] < 0: + if not v[SWEEP_STOP] < v[SWEEP_START]: + raise ValueError( + f"Cannot create sweep with a negative '{SWEEP_JUMP}' " + "value where the end point is smaller than the start." + ) + + def expand_sweeps(self)->list[Tuple[str,Any]]: + """Function to get all combinations of sweep parameters""" + values_dict = {} + # get a collection of a individual sweep values + for var, val in self.sweep.items(): + values_dict[var] = [] + par_val = val[SWEEP_START] + while par_val <= val[SWEEP_STOP]: + values_dict[var].append((var, par_val)) + par_val += val[SWEEP_JUMP] + + # combine all combinations of sweep values + return list(itertools.product( + *[v for v in values_dict.values()])) + class BaseRule: # A unique identifier for the rule diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 0f04a3f..67aa43e 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -38,11 +38,6 @@ _DEFAULT_MASK = [ FILE_RETROACTIVE_EVENT ] -# Parameter sweep keys -SWEEP_START = "start" -SWEEP_STOP = "stop" -SWEEP_JUMP = "jump" - class FileEventPattern(BasePattern): # The path at which events will trigger this pattern triggering_path:str @@ -50,25 +45,19 @@ class FileEventPattern(BasePattern): triggering_file:str # Which types of event the pattern responds to event_mask:list[str] - # TODO move me to BasePattern defintion - # A collection of variables to be swept over for job scheduling - sweep:dict[str,Any] - def __init__(self, name:str, triggering_path:str, recipe:str, triggering_file:str, event_mask:list[str]=_DEFAULT_MASK, parameters:dict[str,Any]={}, outputs:dict[str,Any]={}, sweep:dict[str,Any]={}): """FileEventPattern Constructor. This is used to match against file system events, as caught by the python watchdog module.""" - super().__init__(name, recipe, parameters, outputs) + super().__init__(name, recipe, parameters, outputs, sweep) self._is_valid_triggering_path(triggering_path) self.triggering_path = triggering_path self._is_valid_triggering_file(triggering_file) self.triggering_file = triggering_file self._is_valid_event_mask(event_mask) self.event_mask = event_mask - self._is_valid_sweep(sweep) - self.sweep = sweep def _is_valid_triggering_path(self, triggering_path:str)->None: """Validation check for 'triggering_path' variable from main @@ -112,40 +101,9 @@ class FileEventPattern(BasePattern): raise ValueError(f"Invalid event mask '{mask}'. Valid are: " f"{FILE_EVENTS}") - def _is_valid_sweep(self, sweep)->None: + def _is_valid_sweep(self, sweep: dict[str,Union[int,float,complex]]) -> None: """Validation check for 'sweep' variable from main constructor.""" - check_type(sweep, dict) - if not sweep: - return - for k, v in sweep.items(): - valid_dict( - v, str, Any, [ - SWEEP_START, SWEEP_STOP, SWEEP_JUMP - ], strict=True) - - check_type( - v[SWEEP_START], expected_type=int, alt_types=[float, complex]) - check_type( - v[SWEEP_STOP], expected_type=int, alt_types=[float, complex]) - check_type( - v[SWEEP_JUMP], expected_type=int, alt_types=[float, complex]) - # Try to check that this loop is not infinite - if v[SWEEP_JUMP] == 0: - raise ValueError( - f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero" - ) - elif v[SWEEP_JUMP] > 0: - if not v[SWEEP_STOP] > v[SWEEP_START]: - raise ValueError( - f"Cannot create sweep with a positive '{SWEEP_JUMP}' " - "value where the end point is smaller than the start." - ) - elif v[SWEEP_JUMP] < 0: - if not v[SWEEP_STOP] < v[SWEEP_START]: - raise ValueError( - f"Cannot create sweep with a negative '{SWEEP_JUMP}' " - "value where the end point is smaller than the start." - ) + return super()._is_valid_sweep(sweep) class WatchdogMonitor(BaseMonitor): diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index 60a8606..3f934d9 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -19,11 +19,11 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \ EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \ PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \ - PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, EVENT_RULE + PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \ + EVENT_RULE from core.functionality import print_debug, create_job, replace_keywords, \ make_dir, write_yaml, write_notebook from core.meow import BaseRecipe, BaseHandler -from patterns.file_event_pattern import SWEEP_START, SWEEP_STOP, SWEEP_JUMP class JupyterNotebookRecipe(BaseRecipe): @@ -107,17 +107,7 @@ class PapermillHandler(BaseHandler): self.setup_job(event, yaml_dict) else: # If parameter sweeps, then many jobs created - values_dict = {} - for var, val in rule.pattern.sweep.items(): - values_dict[var] = [] - par_val = val[SWEEP_START] - while par_val <= val[SWEEP_STOP]: - values_dict[var].append((var, par_val)) - par_val += val[SWEEP_JUMP] - - # combine all combinations of sweep values - values_list = list(itertools.product( - *[v for v in values_dict.values()])) + values_list = rule.pattern.expand_sweeps() for values in values_list: for value in values: yaml_dict[value[0]] = value[1] @@ -136,7 +126,6 @@ class PapermillHandler(BaseHandler): pass return False, str(e) - def _is_valid_handler_base(self, handler_base)->None: """Validation check for 'handler_base' variable from main constructor.""" diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index a5352b5..7e9b613 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -22,7 +22,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ from core.functionality import print_debug, create_job, replace_keywords, \ make_dir, write_yaml, write_notebook from core.meow import BaseRecipe, BaseHandler -from patterns.file_event_pattern import SWEEP_START, SWEEP_STOP, SWEEP_JUMP class PythonRecipe(BaseRecipe): @@ -98,17 +97,7 @@ class PythonHandler(BaseHandler): self.setup_job(event, yaml_dict) else: # If parameter sweeps, then many jobs created - values_dict = {} - for var, val in rule.pattern.sweep.items(): - values_dict[var] = [] - par_val = val[SWEEP_START] - while par_val <= val[SWEEP_STOP]: - values_dict[var].append((var, par_val)) - par_val += val[SWEEP_JUMP] - - # combine all combinations of sweep values - values_list = list(itertools.product( - *[v for v in values_dict.values()])) + values_list = rule.pattern.expand_sweeps() for values in values_list: for value in values: yaml_dict[value[0]] = value[1] diff --git a/tests/test_meow.py b/tests/test_meow.py index 7970344..081d75b 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -65,7 +65,10 @@ class MeowTests(unittest.TestCase): pass def _is_valid_output(self, outputs:Any)->None: pass - FullPattern("name", "", "", "") + def _is_valid_sweep(self, + sweep:dict[str,Union[int,float,complex]])->None: + pass + FullPattern("name", "", "", "", "") # Test that BaseRecipe instantiation def testBaseRule(self)->None: @@ -224,3 +227,5 @@ class MeowTests(unittest.TestCase): pass FullTestConductor() + + # TODO Test expansion of parameter sweeps diff --git a/tests/test_patterns.py b/tests/test_patterns.py index eee2b74..2456edd 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -6,10 +6,11 @@ import unittest from multiprocessing import Pipe from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \ - EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH + EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \ + SWEEP_JUMP, SWEEP_STOP from core.functionality import make_dir from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \ - _DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP + _DEFAULT_MASK from recipes import JupyterNotebookRecipe from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE diff --git a/tests/test_recipes.py b/tests/test_recipes.py index a973423..8c78977 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -9,13 +9,12 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \ JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \ PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \ - PARAMS_FILE, JOB_FILE, RESULT_FILE + PARAMS_FILE, JOB_FILE, RESULT_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START from core.correctness.validation import valid_job from core.functionality import get_file_hash, create_job, \ create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml from core.meow import create_rules, create_rule -from patterns.file_event_pattern import FileEventPattern, SWEEP_START, \ - SWEEP_STOP, SWEEP_JUMP +from patterns.file_event_pattern import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler, job_func from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule