NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

moved sweep definitions to base pattern

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-03 10:47:51 +01:00
parent 64aaf46196
commit 1b638ec496
8 changed files with 83 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/correctness/vars.py
View File

@ -113,6 +113,11 @@ PARAMS_FILE = "params.yml"
JOB_FILE = "job.ipynb"
RESULT_FILE = "result.ipynb"
# Parameter sweep keys
SWEEP_START = "start"
SWEEP_STOP = "stop"
SWEEP_JUMP = "jump"
# debug printing levels
DEBUG_ERROR = 1
DEBUG_WARNING = 2

62
core/meow.py
View File

@ -8,6 +8,7 @@ processing.
Author(s): David Marchant
"""
import inspect
import itertools
import sys
from copy import deepcopy
@ -15,7 +16,7 @@ from typing import Any, Union, Tuple
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
get_drt_imp_msg
SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
from core.correctness.validation import valid_string, check_type, \
check_implementation, valid_list, valid_dict
from core.functionality import generate_id
@ -86,14 +87,18 @@ class BasePattern:
parameters:dict[str,Any]
# Parameters showing the potential outputs of a recipe
outputs:dict[str,Any]
# A collection of variables to be swept over for job scheduling
sweep:dict[str,Any]
def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={},
outputs:dict[str,Any]={}):
outputs:dict[str,Any]={}, sweep:dict[str,Any]={}):
"""BasePattern Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_recipe, BasePattern)
check_implementation(type(self)._is_valid_parameters, BasePattern)
check_implementation(type(self)._is_valid_output, BasePattern)
check_implementation(type(self)._is_valid_sweep, BasePattern)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
@ -102,6 +107,8 @@ class BasePattern:
self.parameters = parameters
self._is_valid_output(outputs)
self.outputs = outputs
self._is_valid_sweep(sweep)
self.sweep = sweep
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
@ -132,6 +139,57 @@ class BasePattern:
be implemented by any child class."""
pass
def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None:
"""Validation check for 'sweep' variable from main constructor. Must
be implemented by any child class."""
check_type(sweep, dict)
if not sweep:
return
for _, v in sweep.items():
valid_dict(
v, str, Any, [
SWEEP_START, SWEEP_STOP, SWEEP_JUMP
], strict=True)
check_type(
v[SWEEP_START], expected_type=int, alt_types=[float, complex])
check_type(
v[SWEEP_STOP], expected_type=int, alt_types=[float, complex])
check_type(
v[SWEEP_JUMP], expected_type=int, alt_types=[float, complex])
# Try to check that this loop is not infinite
if v[SWEEP_JUMP] == 0:
raise ValueError(
f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
)
elif v[SWEEP_JUMP] > 0:
if not v[SWEEP_STOP] > v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
elif v[SWEEP_JUMP] < 0:
if not v[SWEEP_STOP] < v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
def expand_sweeps(self)->list[Tuple[str,Any]]:
"""Function to get all combinations of sweep parameters"""
values_dict = {}
# get a collection of a individual sweep values
for var, val in self.sweep.items():
values_dict[var] = []
par_val = val[SWEEP_START]
while par_val <= val[SWEEP_STOP]:
values_dict[var].append((var, par_val))
par_val += val[SWEEP_JUMP]
# combine all combinations of sweep values
return list(itertools.product(
*[v for v in values_dict.values()]))
class BaseRule:
# A unique identifier for the rule

48
patterns/file_event_pattern.py
View File

@ -38,11 +38,6 @@ _DEFAULT_MASK = [
FILE_RETROACTIVE_EVENT
]
# Parameter sweep keys
SWEEP_START = "start"
SWEEP_STOP = "stop"
SWEEP_JUMP = "jump"
class FileEventPattern(BasePattern):
# The path at which events will trigger this pattern
triggering_path:str
@ -50,25 +45,19 @@ class FileEventPattern(BasePattern):
triggering_file:str
# Which types of event the pattern responds to
event_mask:list[str]
# TODO move me to BasePattern defintion
# A collection of variables to be swept over for job scheduling
sweep:dict[str,Any]
def __init__(self, name:str, triggering_path:str, recipe:str,
triggering_file:str, event_mask:list[str]=_DEFAULT_MASK,
parameters:dict[str,Any]={}, outputs:dict[str,Any]={},
sweep:dict[str,Any]={}):
"""FileEventPattern Constructor. This is used to match against file
system events, as caught by the python watchdog module."""
super().__init__(name, recipe, parameters, outputs)
super().__init__(name, recipe, parameters, outputs, sweep)
self._is_valid_triggering_path(triggering_path)
self.triggering_path = triggering_path
self._is_valid_triggering_file(triggering_file)
self.triggering_file = triggering_file
self._is_valid_event_mask(event_mask)
self.event_mask = event_mask
self._is_valid_sweep(sweep)
self.sweep = sweep
def _is_valid_triggering_path(self, triggering_path:str)->None:
"""Validation check for 'triggering_path' variable from main
@ -112,40 +101,9 @@ class FileEventPattern(BasePattern):
raise ValueError(f"Invalid event mask '{mask}'. Valid are: "
f"{FILE_EVENTS}")
def _is_valid_sweep(self, sweep)->None:
def _is_valid_sweep(self, sweep: dict[str,Union[int,float,complex]]) -> None:
"""Validation check for 'sweep' variable from main constructor."""
check_type(sweep, dict)
if not sweep:
return
for k, v in sweep.items():
valid_dict(
v, str, Any, [
SWEEP_START, SWEEP_STOP, SWEEP_JUMP
], strict=True)
check_type(
v[SWEEP_START], expected_type=int, alt_types=[float, complex])
check_type(
v[SWEEP_STOP], expected_type=int, alt_types=[float, complex])
check_type(
v[SWEEP_JUMP], expected_type=int, alt_types=[float, complex])
# Try to check that this loop is not infinite
if v[SWEEP_JUMP] == 0:
raise ValueError(
f"Cannot create sweep with a '{SWEEP_JUMP}' value of zero"
)
elif v[SWEEP_JUMP] > 0:
if not v[SWEEP_STOP] > v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a positive '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
elif v[SWEEP_JUMP] < 0:
if not v[SWEEP_STOP] < v[SWEEP_START]:
raise ValueError(
f"Cannot create sweep with a negative '{SWEEP_JUMP}' "
"value where the end point is smaller than the start."
)
return super()._is_valid_sweep(sweep)
class WatchdogMonitor(BaseMonitor):

17
recipes/jupyter_notebook_recipe.py
View File

@ -19,11 +19,11 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \
EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, EVENT_RULE
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
EVENT_RULE
from core.functionality import print_debug, create_job, replace_keywords, \
make_dir, write_yaml, write_notebook
from core.meow import BaseRecipe, BaseHandler
from patterns.file_event_pattern import SWEEP_START, SWEEP_STOP, SWEEP_JUMP
class JupyterNotebookRecipe(BaseRecipe):
@ -107,17 +107,7 @@ class PapermillHandler(BaseHandler):
self.setup_job(event, yaml_dict)
else:
# If parameter sweeps, then many jobs created
values_dict = {}
for var, val in rule.pattern.sweep.items():
values_dict[var] = []
par_val = val[SWEEP_START]
while par_val <= val[SWEEP_STOP]:
values_dict[var].append((var, par_val))
par_val += val[SWEEP_JUMP]
# combine all combinations of sweep values
values_list = list(itertools.product(
*[v for v in values_dict.values()]))
values_list = rule.pattern.expand_sweeps()
for values in values_list:
for value in values:
yaml_dict[value[0]] = value[1]
@ -136,7 +126,6 @@ class PapermillHandler(BaseHandler):
pass
return False, str(e)
def _is_valid_handler_base(self, handler_base)->None:
"""Validation check for 'handler_base' variable from main
constructor."""

13
recipes/python_recipe.py
View File

@ -22,7 +22,6 @@ from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
from core.functionality import print_debug, create_job, replace_keywords, \
make_dir, write_yaml, write_notebook
from core.meow import BaseRecipe, BaseHandler
from patterns.file_event_pattern import SWEEP_START, SWEEP_STOP, SWEEP_JUMP
class PythonRecipe(BaseRecipe):
@ -98,17 +97,7 @@ class PythonHandler(BaseHandler):
self.setup_job(event, yaml_dict)
else:
# If parameter sweeps, then many jobs created
values_dict = {}
for var, val in rule.pattern.sweep.items():
values_dict[var] = []
par_val = val[SWEEP_START]
while par_val <= val[SWEEP_STOP]:
values_dict[var].append((var, par_val))
par_val += val[SWEEP_JUMP]
# combine all combinations of sweep values
values_list = list(itertools.product(
*[v for v in values_dict.values()]))
values_list = rule.pattern.expand_sweeps()
for values in values_list:
for value in values:
yaml_dict[value[0]] = value[1]

7
tests/test_meow.py
View File

@ -65,7 +65,10 @@ class MeowTests(unittest.TestCase):
pass
def _is_valid_output(self, outputs:Any)->None:
pass
FullPattern("name", "", "", "")
def _is_valid_sweep(self,
sweep:dict[str,Union[int,float,complex]])->None:
pass
FullPattern("name", "", "", "", "")
# Test that BaseRecipe instantiation
def testBaseRule(self)->None:
@ -224,3 +227,5 @@ class MeowTests(unittest.TestCase):
pass
FullTestConductor()
# TODO Test expansion of parameter sweeps

5
tests/test_patterns.py
View File

@ -6,10 +6,11 @@ import unittest
from multiprocessing import Pipe
from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
SWEEP_JUMP, SWEEP_STOP
from core.functionality import make_dir
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
_DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP
_DEFAULT_MASK
from recipes import JupyterNotebookRecipe
from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE

5
tests/test_recipes.py
View File

@ -9,13 +9,12 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \
PARAMS_FILE, JOB_FILE, RESULT_FILE
PARAMS_FILE, JOB_FILE, RESULT_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from core.correctness.validation import valid_job
from core.functionality import get_file_hash, create_job, \
create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml
from core.meow import create_rules, create_rule
from patterns.file_event_pattern import FileEventPattern, SWEEP_START, \
SWEEP_STOP, SWEEP_JUMP
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, job_func
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule