moved sweep definitions to base pattern

core/correctness/vars.py

@@ -113,6 +113,11 @@ PARAMS_FILE = "params.yml"
 JOB_FILE = "job.ipynb"
 RESULT_FILE = "result.ipynb"
 
+# Parameter sweep keys
+SWEEP_START = "start"
+SWEEP_STOP = "stop"
+SWEEP_JUMP = "jump"
+
 # debug printing levels
 DEBUG_ERROR = 1
 DEBUG_WARNING = 2

core/meow.py

@@ -8,6 +8,7 @@ processing.
 Author(s): David Marchant
 """
 import inspect
+import itertools
 import sys
 
 from copy import deepcopy
@@ -15,7 +16,7 @@ from typing import Any, Union, Tuple
 
 from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
     VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
-    get_drt_imp_msg
+    SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
 from core.correctness.validation import valid_string, check_type, \
     check_implementation, valid_list, valid_dict
 from core.functionality import generate_id
@@ -86,14 +87,18 @@ class BasePattern:
     parameters:dict[str,Any]
     # Parameters showing the potential outputs of a recipe
     outputs:dict[str,Any]
+    # A collection of variables to be swept over for job scheduling
+    sweep:dict[str,Any]
+
     def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={}, 
-            outputs:dict[str,Any]={}):
+            outputs:dict[str,Any]={}, sweep:dict[str,Any]={}):
         """BasePattern Constructor. This will check that any class inheriting 
         from it implements its validation functions. It will then call these on
         the input parameters."""
         check_implementation(type(self)._is_valid_recipe, BasePattern)
         check_implementation(type(self)._is_valid_parameters, BasePattern)
         check_implementation(type(self)._is_valid_output, BasePattern)
+        check_implementation(type(self)._is_valid_sweep, BasePattern)
         self._is_valid_name(name)
         self.name = name
         self._is_valid_recipe(recipe)
@@ -102,6 +107,8 @@ class BasePattern:
         self.parameters = parameters
         self._is_valid_output(outputs)
         self.outputs = outputs
+        self._is_valid_sweep(sweep)
+        self.sweep = sweep
 
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 
@@ -132,6 +139,57 @@ class BasePattern:
         be implemented by any child class."""
         pass
 
+    def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None:
+        """Validation check for 'sweep' variable from main constructor. Must 
+        be implemented by any child class."""
+        check_type(sweep, dict)
+        if not sweep:
+            return
+        for _, v in sweep.items():
+            valid_dict(
+                v, str, Any, [
+                    SWEEP_START, SWEEP_STOP, SWEEP_JUMP
+                ], strict=True)
+
+            check_type(
+                v[SWEEP_START], expected_type=int, alt_types=[float, complex])
+            check_type(
+                v[SWEEP_STOP], expected_type=int, alt_types=[float, complex])
+            check_type(
+                v[SWEEP_JUMP], expected_type=int, alt_types=[float, complex])
+            # Try to check that this loop is not infinite
+            if v[SWEEP_JUMP] == 0:
+                raise ValueError(
+                    f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
+                )
+            elif v[SWEEP_JUMP] > 0:
+                if not v[SWEEP_STOP] > v[SWEEP_START]:
+                    raise ValueError(
+                        f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
+                        "value where the end point is smaller than the start."
+                    )
+            elif v[SWEEP_JUMP] < 0:
+                if not v[SWEEP_STOP] < v[SWEEP_START]:
+                    raise ValueError(
+                        f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
+                        "value where the end point is smaller than the start."
+                    )
+
+    def expand_sweeps(self)->list[Tuple[str,Any]]:
+        """Function to get all combinations of sweep parameters"""
+        values_dict = {}
+        # get a collection of a individual sweep values
+        for var, val in self.sweep.items():
+            values_dict[var] = []
+            par_val = val[SWEEP_START]
+            while par_val <= val[SWEEP_STOP]:
+                values_dict[var].append((var, par_val))
+                par_val += val[SWEEP_JUMP]
+
+        # combine all combinations of sweep values
+        return list(itertools.product(
+            *[v for v in values_dict.values()]))
+
 
 class BaseRule:
     # A unique identifier for the rule

patterns/file_event_pattern.py

@@ -38,11 +38,6 @@ _DEFAULT_MASK = [
     FILE_RETROACTIVE_EVENT
 ]
 
-# Parameter sweep keys
-SWEEP_START = "start"
-SWEEP_STOP = "stop"
-SWEEP_JUMP = "jump"
-
 class FileEventPattern(BasePattern):
     # The path at which events will trigger this pattern
     triggering_path:str
@@ -50,25 +45,19 @@ class FileEventPattern(BasePattern):
     triggering_file:str
     # Which types of event the pattern responds to
     event_mask:list[str]
-    # TODO move me to BasePattern defintion
-    # A collection of variables to be swept over for job scheduling
-    sweep:dict[str,Any]
-
     def __init__(self, name:str, triggering_path:str, recipe:str, 
             triggering_file:str, event_mask:list[str]=_DEFAULT_MASK, 
             parameters:dict[str,Any]={}, outputs:dict[str,Any]={}, 
             sweep:dict[str,Any]={}):
         """FileEventPattern Constructor. This is used to match against file 
         system events, as caught by the python watchdog module."""
-        super().__init__(name, recipe, parameters, outputs)
+        super().__init__(name, recipe, parameters, outputs, sweep)
         self._is_valid_triggering_path(triggering_path)
         self.triggering_path = triggering_path
         self._is_valid_triggering_file(triggering_file)
         self.triggering_file = triggering_file
         self._is_valid_event_mask(event_mask)
         self.event_mask = event_mask
-        self._is_valid_sweep(sweep)
-        self.sweep = sweep
 
     def _is_valid_triggering_path(self, triggering_path:str)->None:
         """Validation check for 'triggering_path' variable from main 
@@ -112,40 +101,9 @@ class FileEventPattern(BasePattern):
                 raise ValueError(f"Invalid event mask '{mask}'. Valid are: "
                     f"{FILE_EVENTS}")
 
-    def _is_valid_sweep(self, sweep)->None:
+    def _is_valid_sweep(self, sweep: dict[str,Union[int,float,complex]]) -> None:
         """Validation check for 'sweep' variable from main constructor."""
-        check_type(sweep, dict)
+        return super()._is_valid_sweep(sweep)
-        if not sweep:
-            return
-        for k, v in sweep.items():
-            valid_dict(
-                v, str, Any, [
-                    SWEEP_START, SWEEP_STOP, SWEEP_JUMP
-                ], strict=True)
-
-            check_type(
-                v[SWEEP_START], expected_type=int, alt_types=[float, complex])
-            check_type(
-                v[SWEEP_STOP], expected_type=int, alt_types=[float, complex])
-            check_type(
-                v[SWEEP_JUMP], expected_type=int, alt_types=[float, complex])
-            # Try to check that this loop is not infinite
-            if v[SWEEP_JUMP] == 0:
-                raise ValueError(
-                    f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
-                )
-            elif v[SWEEP_JUMP] > 0:
-                if not v[SWEEP_STOP] > v[SWEEP_START]:
-                    raise ValueError(
-                        f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
-                        "value where the end point is smaller than the start."
-                    )
-            elif v[SWEEP_JUMP] < 0:
-                if not v[SWEEP_STOP] < v[SWEEP_START]:
-                    raise ValueError(
-                        f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
-                        "value where the end point is smaller than the start."
-                    )
 
 
 class WatchdogMonitor(BaseMonitor):

recipes/jupyter_notebook_recipe.py

@@ -19,11 +19,11 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \
     EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
     PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \
-    PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, EVENT_RULE
+    PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
+    EVENT_RULE
 from core.functionality import print_debug, create_job, replace_keywords, \
     make_dir, write_yaml, write_notebook
 from core.meow import BaseRecipe, BaseHandler
-from patterns.file_event_pattern import SWEEP_START, SWEEP_STOP, SWEEP_JUMP
 
 
 class JupyterNotebookRecipe(BaseRecipe):
@@ -107,17 +107,7 @@ class PapermillHandler(BaseHandler):
             self.setup_job(event, yaml_dict)
         else:
             # If parameter sweeps, then many jobs created
-            values_dict = {}
+            values_list = rule.pattern.expand_sweeps()
-            for var, val in rule.pattern.sweep.items():
-                values_dict[var] = []
-                par_val = val[SWEEP_START]
-                while par_val <= val[SWEEP_STOP]:
-                    values_dict[var].append((var, par_val))
-                    par_val += val[SWEEP_JUMP]
-
-            # combine all combinations of sweep values
-            values_list = list(itertools.product(
-                *[v for v in values_dict.values()]))
             for values in values_list:
                 for value in values:
                     yaml_dict[value[0]] = value[1]
@@ -136,7 +126,6 @@ class PapermillHandler(BaseHandler):
             pass
         return False, str(e)
 
-
     def _is_valid_handler_base(self, handler_base)->None:
         """Validation check for 'handler_base' variable from main 
         constructor."""

recipes/python_recipe.py

@@ -22,7 +22,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
 from core.functionality import print_debug, create_job, replace_keywords, \
     make_dir, write_yaml, write_notebook
 from core.meow import BaseRecipe, BaseHandler
-from patterns.file_event_pattern import SWEEP_START, SWEEP_STOP, SWEEP_JUMP
 
 
 class PythonRecipe(BaseRecipe):
@@ -98,17 +97,7 @@ class PythonHandler(BaseHandler):
             self.setup_job(event, yaml_dict)
         else:
             # If parameter sweeps, then many jobs created
-            values_dict = {}
+            values_list = rule.pattern.expand_sweeps()
-            for var, val in rule.pattern.sweep.items():
-                values_dict[var] = []
-                par_val = val[SWEEP_START]
-                while par_val <= val[SWEEP_STOP]:
-                    values_dict[var].append((var, par_val))
-                    par_val += val[SWEEP_JUMP]
-
-            # combine all combinations of sweep values
-            values_list = list(itertools.product(
-                *[v for v in values_dict.values()]))
             for values in values_list:
                 for value in values:
                     yaml_dict[value[0]] = value[1]

tests/test_meow.py

@@ -65,7 +65,10 @@ class MeowTests(unittest.TestCase):
                 pass
             def _is_valid_output(self, outputs:Any)->None:
                 pass
-        FullPattern("name", "", "", "")
+            def _is_valid_sweep(self, 
+                    sweep:dict[str,Union[int,float,complex]])->None:
+                pass
+        FullPattern("name", "", "", "", "")
 
     # Test that BaseRecipe instantiation
     def testBaseRule(self)->None:
@@ -224,3 +227,5 @@ class MeowTests(unittest.TestCase):
                 pass
 
         FullTestConductor()
+
+    # TODO Test expansion of parameter sweeps

tests/test_patterns.py

@@ -6,10 +6,11 @@ import unittest
 from multiprocessing import Pipe
 
 from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
-    EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH
+    EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
+    SWEEP_JUMP, SWEEP_STOP
 from core.functionality import make_dir
 from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
-    _DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP
+    _DEFAULT_MASK
 from recipes import JupyterNotebookRecipe
 from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE
 

tests/test_recipes.py

@@ -9,13 +9,12 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
     EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
     JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
     PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \
-    PARAMS_FILE, JOB_FILE, RESULT_FILE
+    PARAMS_FILE, JOB_FILE, RESULT_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START
 from core.correctness.validation import valid_job
 from core.functionality import get_file_hash, create_job, \
     create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml
 from core.meow import create_rules, create_rule
-from patterns.file_event_pattern import FileEventPattern, SWEEP_START, \
+from patterns.file_event_pattern import FileEventPattern
-    SWEEP_STOP, SWEEP_JUMP
 from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
     PapermillHandler, job_func
 from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule