NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated code so should work on windows, with exception of waiting on multiple connections

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-08 14:46:03 +01:00
parent d787e37adc
commit a1451881ae
17 changed files with 280 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -19,3 +19,8 @@ Pytest unittests are provided within the 'tests directory, as well as a script *
with individual tests runnable with:
pytest test_runner.py::MeowTests::testMeowRunnerLinkedPythonExecution -W ignore::DeprecationWarning
to run locally, update your '~/.bashrc' file to inclue:
# Manually added to get local testing of Python files working easier
export PYTHONPATH=/home/patch/Documents/Research/Python

6
conductors/local_python_conductor.py
View File

@ -9,7 +9,7 @@ import os
import shutil
from datetime import datetime
from typing import Any, Tuple
from typing import Any, Tuple, Dict
from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
@ -25,7 +25,7 @@ class LocalPythonConductor(BaseConductor):
def __init__(self)->None:
super().__init__()
def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]:
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
process it or not. This conductor will accept any Python job type"""
try:
@ -36,7 +36,7 @@ class LocalPythonConductor(BaseConductor):
pass
return False, str(e)
def execute(self, job:dict[str,Any])->None:
def execute(self, job:Dict[str,Any])->None:
valid_job(job)
job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID])

40
core/correctness/validation.py
View File

@ -8,7 +8,8 @@ Author(s): David Marchant
from datetime import datetime
from inspect import signature
from os.path import sep, exists, isfile, isdir, dirname
from typing import Any, _SpecialForm, Union, Tuple, get_origin, get_args
from typing import Any, _SpecialForm, Union, Tuple, Type, Dict, List, \
get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
@ -33,7 +34,7 @@ WATCHDOG_EVENT_KEYS = {
# Required keys in job dict
JOB_KEYS = {
JOB_TYPE: str,
JOB_EVENT: dict,
JOB_EVENT: Dict,
JOB_ID: str,
JOB_PATTERN: Any,
JOB_RECIPE: Any,
@ -42,7 +43,7 @@ JOB_KEYS = {
JOB_CREATE_TIME: datetime,
}
def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
or_none:bool=False)->None:
"""Checks if a given variable is of the expected type. Raises TypeError or
ValueError as appropriate if any issues are encountered."""
@ -52,6 +53,11 @@ def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
if get_origin(expected_type) is Union:
type_list = list(get_args(expected_type))
type_list = type_list + alt_types
# # If we have any types from typing, then update to allow checks against
# # their base types too
# for t in type_list:
# if get_origin(t):
# type_list.append(get_origin(t))
# Only accept None if explicitly allowed
if variable is None:
@ -130,16 +136,16 @@ def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
"%s" % (char, valid_chars)
)
def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type,
required_keys:list[Any]=[], optional_keys:list[Any]=[],
def valid_dict(variable:Dict[Any, Any], key_type:Type, value_type:Type,
required_keys:List[Any]=[], optional_keys:List[Any]=[],
strict:bool=True, min_length:int=1)->None:
"""Checks that a given dictionary is valid. Key and Value types are
enforced, as are required and optional keys. Will raise ValueError,
TypeError or KeyError depending on the problem encountered."""
# Validate inputs
check_type(variable, dict)
check_type(key_type, type, alt_types=[_SpecialForm])
check_type(value_type, type, alt_types=[_SpecialForm])
check_type(variable, Dict)
check_type(key_type, Type, alt_types=[_SpecialForm])
check_type(value_type, Type, alt_types=[_SpecialForm])
check_type(required_keys, list)
check_type(optional_keys, list)
check_type(strict, bool)
@ -172,11 +178,11 @@ def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type,
raise ValueError(f"Unexpected key '{k}' should not be present "
f"in dict '{variable}'")
def valid_list(variable:list[Any], entry_type:type,
alt_types:list[type]=[], min_length:int=1)->None:
def valid_list(variable:List[Any], entry_type:Type,
alt_types:List[Type]=[], min_length:int=1)->None:
"""Checks that a given list is valid. Value types are checked and a
ValueError or TypeError is raised if a problem is encountered."""
check_type(variable, list)
check_type(variable, List)
# Check length meets minimum
if len(variable) < min_length:
@ -256,11 +262,11 @@ def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:
return print, logging
def valid_meow_dict(meow_dict:dict[str,Any], msg:str,
keys:dict[str,type])->None:
def valid_meow_dict(meow_dict:Dict[str,Any], msg:str,
keys:Dict[str,Type])->None:
"""Check given dictionary expresses a meow construct. This won't do much
directly, but is called by more specific validation functions."""
check_type(meow_dict, dict)
check_type(meow_dict, Dict)
# Check we have all the required keys, and they are all of the expected
# type
for key, value_type in keys.items():
@ -268,13 +274,13 @@ def valid_meow_dict(meow_dict:dict[str,Any], msg:str,
raise KeyError(f"{msg} require key '{key}'")
check_type(meow_dict[key], value_type)
def valid_event(event:dict[str,Any])->None:
def valid_event(event:Dict[str,Any])->None:
"""Check that a given dict expresses a meow event."""
valid_meow_dict(event, "Event", EVENT_KEYS)
def valid_job(job:dict[str,Any])->None:
def valid_job(job:Dict[str,Any])->None:
"""Check that a given dict expresses a meow job."""
valid_meow_dict(job, "Job", JOB_KEYS)
def valid_watchdog_event(event:dict[str,Any])->None:
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

44
core/functionality.py
View File

@ -7,11 +7,12 @@ import os
import yaml
from datetime import datetime
from typing import List
from multiprocessing.connection import Connection, wait as multi_wait
from multiprocessing.queues import Queue
from papermill.translators import papermill_translators
from typing import Any
from typing import Any, Dict
from random import SystemRandom
from core.correctness.validation import check_type, valid_existing_file_path, \
@ -36,7 +37,7 @@ KEYWORD_JOB = "{JOB}"
#TODO Make this guaranteed unique
def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
def generate_id(prefix:str="", length:int=16, existing_ids:List[str]=[],
charset:str=CHAR_UPPERCASE+CHAR_LOWERCASE, attempts:int=24):
random_length = max(length - len(prefix), 0)
for _ in range(attempts):
@ -47,10 +48,12 @@ def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
raise ValueError(f"Could not generate ID unique from '{existing_ids}' "
f"using values '{charset}' and length of '{length}'.")
def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]:
def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:
all_connections = [i for i in inputs if type(i) is Connection] \
+ [i._reader for i in inputs if type(i) is Queue]
for i in inputs:
print(type(i))
ready = multi_wait(all_connections)
ready_inputs = [i for i in inputs if \
(type(i) is Connection and i in ready) \
@ -72,7 +75,6 @@ def _get_file_sha256(file_path):
def get_file_hash(file_path:str, hash:str):
check_type(hash, str)
import os
valid_existing_file_path(file_path)
valid_hashes = {
@ -80,7 +82,7 @@ def get_file_hash(file_path:str, hash:str):
}
if hash not in valid_hashes:
raise KeyError(f"Cannot use hash '{hash}'. Valid are "
"'{list(valid_hashes.keys())}")
f"'{list(valid_hashes.keys())}")
return valid_hashes[hash](file_path)
@ -169,7 +171,7 @@ def read_notebook(filepath:str):
with open(filepath, 'r') as read_file:
return json.load(read_file)
def write_notebook(source:dict[str,Any], filename:str):
def write_notebook(source:Dict[str,Any], filename:str):
"""
Writes the given notebook source code to a given filename.
@ -183,10 +185,10 @@ def write_notebook(source:dict[str,Any], filename:str):
json.dump(source, job_file)
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any],
parameters:dict[str,Any], expand_env_values:bool=False)->dict[str,Any]:
def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any],
parameters:Dict[str,Any], expand_env_values:bool=False)->Dict[str,Any]:
nbformat.validate(jupyter_notebook)
check_type(parameters, dict)
check_type(parameters, Dict)
if jupyter_notebook["nbformat"] != 4:
raise Warning(
@ -256,10 +258,10 @@ def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any],
return output_notebook
def parameterize_python_script(script:list[str], parameters:dict[str,Any],
expand_env_values:bool=False)->dict[str,Any]:
def parameterize_python_script(script:List[str], parameters:Dict[str,Any],
expand_env_values:bool=False)->Dict[str,Any]:
check_script(script)
check_type(parameters, dict)
check_type(parameters, Dict)
output_script = copy.deepcopy(script)
@ -302,8 +304,8 @@ def print_debug(print_target, debug_level, msg, level)->None:
status = "WARNING"
print(f"{status}: {msg}", file=print_target)
def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
monitor_base:str)->dict[str,str]:
def replace_keywords(old_dict:Dict[str,str], job_id:str, src_path:str,
monitor_base:str)->Dict[str,str]:
"""Function to replace all MEOW magic words in a dictionary with dynamic
values."""
new_dict = {}
@ -332,8 +334,8 @@ def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
return new_dict
def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={}
)->dict[Any,Any]:
def create_event(event_type:str, path:str, rule:Any, extras:Dict[Any,Any]={}
)->Dict[Any,Any]:
"""Function to create a MEOW dictionary."""
return {
**extras,
@ -343,7 +345,7 @@ def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={}
}
def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
extras:dict[Any,Any]={})->dict[Any,Any]:
extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW event dictionary."""
return create_event(
EVENT_TYPE_WATCHDOG,
@ -358,8 +360,8 @@ def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
}
)
def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
)->dict[Any,Any]:
def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
)->Dict[Any,Any]:
"""Function to create a MEOW job dictionary."""
job_dict = {
#TODO compress event?
@ -376,7 +378,7 @@ def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
return {**extras, **job_dict}
def lines_to_string(lines:list[str])->str:
def lines_to_string(lines:List[str])->str:
"""Function to convert a list of str lines, into one continuous string
separated by newline characters"""
return "\n".join(lines)

66
core/meow.py
View File

@ -12,7 +12,7 @@ import itertools
import sys
from copy import deepcopy
from typing import Any, Union, Tuple
from typing import Any, Union, Tuple, Dict, List
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
@ -28,11 +28,11 @@ class BaseRecipe:
# Actual code to run
recipe:Any
# Possible parameters that could be overridden by a Pattern
parameters:dict[str, Any]
parameters:Dict[str, Any]
# Additional configuration options
requirements:dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={},
requirements:dict[str,Any]={}):
requirements:Dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={},
requirements:Dict[str,Any]={}):
"""BaseRecipe Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
@ -84,14 +84,14 @@ class BasePattern:
# An identifier of a recipe
recipe:str
# Parameters to be overridden in the recipe
parameters:dict[str,Any]
parameters:Dict[str,Any]
# Parameters showing the potential outputs of a recipe
outputs:dict[str,Any]
outputs:Dict[str,Any]
# A collection of variables to be swept over for job scheduling
sweep:dict[str,Any]
sweep:Dict[str,Any]
def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={},
outputs:dict[str,Any]={}, sweep:dict[str,Any]={}):
def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={},
outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}):
"""BasePattern Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
@ -138,11 +138,11 @@ class BasePattern:
be implemented by any child class."""
pass
def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None:
def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None:
"""Validation check for 'sweep' variable from main constructor. This
function is implemented to check for the types given in the signature,
and must be overridden if these differ."""
check_type(sweep, dict)
check_type(sweep, Dict)
if not sweep:
return
for _, v in sweep.items():
@ -175,7 +175,7 @@ class BasePattern:
"value where the end point is smaller than the start."
)
def expand_sweeps(self)->list[Tuple[str,Any]]:
def expand_sweeps(self)->List[Tuple[str,Any]]:
"""Function to get all combinations of sweep parameters"""
values_dict = {}
# get a collection of a individual sweep values
@ -259,17 +259,17 @@ class BaseRule:
class BaseMonitor:
# A collection of patterns
_patterns: dict[str, BasePattern]
_patterns: Dict[str, BasePattern]
# A collection of recipes
_recipes: dict[str, BaseRecipe]
_recipes: Dict[str, BaseRecipe]
# A collection of rules derived from _patterns and _recipes
_rules: dict[str, BaseRule]
_rules: Dict[str, BaseRule]
# A channel for sending messages to the runner. Note that this is not
# initialised within the constructor, but within the runner when passed the
# monitor is passed to it.
to_runner: VALID_CHANNELS
def __init__(self, patterns:dict[str,BasePattern],
recipes:dict[str,BaseRecipe])->None:
def __init__(self, patterns:Dict[str,BasePattern],
recipes:Dict[str,BaseRecipe])->None:
"""BaseMonitor Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
@ -302,12 +302,12 @@ class BaseMonitor:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None:
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
"""Validation check for 'patterns' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None:
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
"""Validation check for 'recipes' variable from main constructor. Must
be implemented by any child class."""
pass
@ -337,7 +337,7 @@ class BaseMonitor:
implemented by any child process."""
pass
def get_patterns(self)->dict[str,BasePattern]:
def get_patterns(self)->Dict[str,BasePattern]:
"""Function to get a dictionary of all current pattern definitions.
Must be implemented by any child process."""
pass
@ -357,12 +357,12 @@ class BaseMonitor:
implemented by any child process."""
pass
def get_recipes(self)->dict[str,BaseRecipe]:
def get_recipes(self)->Dict[str,BaseRecipe]:
"""Function to get a dictionary of all current recipe definitions.
Must be implemented by any child process."""
pass
def get_rules(self)->dict[str,BaseRule]:
def get_rules(self)->Dict[str,BaseRule]:
"""Function to get a dictionary of all current rule definitions.
Must be implemented by any child process."""
pass
@ -387,12 +387,12 @@ class BaseHandler:
raise TypeError(msg)
return object.__new__(cls)
def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]:
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. Must be implemented by any child process."""
pass
def handle(self, event:dict[str,Any])->None:
def handle(self, event:Dict[str,Any])->None:
"""Function to handle a given event. Must be implemented by any child
process."""
pass
@ -413,28 +413,28 @@ class BaseConductor:
raise TypeError(msg)
return object.__new__(cls)
def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]:
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
process it or not. Must be implemented by any child process."""
pass
def execute(self, job:dict[str,Any])->None:
def execute(self, job:Dict[str,Any])->None:
"""Function to execute a given job. Must be implemented by any child
process."""
pass
def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
recipes:Union[dict[str,BaseRecipe],list[BaseRecipe]],
new_rules:list[BaseRule]=[])->dict[str,BaseRule]:
def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
"""Function to create any valid rules from a given collection of patterns
and recipes. All inbuilt rule types are considered, with additional
definitions provided through the 'new_rules' variable. Note that any
provided pattern and recipe dictionaries must be keyed with the
corresponding pattern and recipe names."""
# Validation of inputs
check_type(patterns, dict, alt_types=[list])
check_type(recipes, dict, alt_types=[list])
check_type(patterns, Dict, alt_types=[List])
check_type(recipes, Dict, alt_types=[List])
valid_list(new_rules, BaseRule, min_length=0)
# Convert a pattern list to a dictionary
@ -477,7 +477,7 @@ def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
return generated_rules
def create_rule(pattern:BasePattern, recipe:BaseRecipe,
new_rules:list[BaseRule]=[])->BaseRule:
new_rules:List[BaseRule]=[])->BaseRule:
"""Function to create a valid rule from a given pattern and recipe. All
inbuilt rule types are considered, with additional definitions provided
through the 'new_rules' variable."""

34
core/runner.py
View File

@ -12,7 +12,7 @@ import threading
from multiprocessing import Pipe
from random import randrange
from typing import Any, Union
from typing import Any, Union, Dict, List
from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
VALID_CHANNELS, JOB_ID, META_FILE
@ -24,18 +24,18 @@ from core.meow import BaseHandler, BaseMonitor, BaseConductor
class MeowRunner:
# A collection of all monitors in the runner
monitors:list[BaseMonitor]
monitors:List[BaseMonitor]
# A collection of all handlers in the runner
handlers:list[BaseHandler]
handlers:List[BaseHandler]
# A collection of all conductors in the runner
conductors:list[BaseConductor]
conductors:List[BaseConductor]
# A collection of all channels from each monitor
from_monitors: list[VALID_CHANNELS]
from_monitors: List[VALID_CHANNELS]
# A collection of all channels from each handler
from_handlers: list[VALID_CHANNELS]
def __init__(self, monitors:Union[BaseMonitor,list[BaseMonitor]],
handlers:Union[BaseHandler,list[BaseHandler]],
conductors:Union[BaseConductor,list[BaseConductor]],
from_handlers: List[VALID_CHANNELS]
def __init__(self, monitors:Union[BaseMonitor,List[BaseMonitor]],
handlers:Union[BaseHandler,List[BaseHandler]],
conductors:Union[BaseConductor,List[BaseConductor]],
print:Any=sys.stdout, logging:int=0)->None:
"""MeowRunner constructor. This connects all provided monitors,
handlers and conductors according to what events and jobs they produce
@ -178,7 +178,7 @@ class MeowRunner:
]
self.execute_job(conductor, job)
def handle_event(self, handler:BaseHandler, event:dict[str:Any])->None:
def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None:
"""Function for a given handler to handle a given event, without
crashing the runner in the event of a problem."""
print_debug(self._print_target, self.debug_level,
@ -193,7 +193,7 @@ class MeowRunner:
"Something went wrong during handling for event "
f"'{event[EVENT_TYPE]}'. {e}", DEBUG_INFO)
def execute_job(self, conductor:BaseConductor, job:dict[str:Any])->None:
def execute_job(self, conductor:BaseConductor, job:Dict[str,Any])->None:
"""Function for a given conductor to execute a given job, without
crashing the runner in the event of a problem."""
print_debug(self._print_target, self.debug_level,
@ -306,22 +306,22 @@ class MeowRunner:
"Job conductor thread stopped", DEBUG_INFO)
def _is_valid_monitors(self,
monitors:Union[BaseMonitor,list[BaseMonitor]])->None:
monitors:Union[BaseMonitor,List[BaseMonitor]])->None:
"""Validation check for 'monitors' variable from main constructor."""
check_type(monitors, BaseMonitor, alt_types=[list])
check_type(monitors, BaseMonitor, alt_types=[List])
if type(monitors) == list:
valid_list(monitors, BaseMonitor, min_length=1)
def _is_valid_handlers(self,
handlers:Union[BaseHandler,list[BaseHandler]])->None:
handlers:Union[BaseHandler,List[BaseHandler]])->None:
"""Validation check for 'handlers' variable from main constructor."""
check_type(handlers, BaseHandler, alt_types=[list])
check_type(handlers, BaseHandler, alt_types=[List])
if type(handlers) == list:
valid_list(handlers, BaseHandler, min_length=1)
def _is_valid_conductors(self,
conductors:Union[BaseConductor,list[BaseConductor]])->None:
conductors:Union[BaseConductor,List[BaseConductor]])->None:
"""Validation check for 'conductors' variable from main constructor."""
check_type(conductors, BaseConductor, alt_types=[list])
check_type(conductors, BaseConductor, alt_types=[List])
if type(conductors) == list:
valid_list(conductors, BaseConductor, min_length=1)

37
patterns/file_event_pattern.py
View File

@ -14,7 +14,7 @@ from copy import deepcopy
from fnmatch import translate
from re import match
from time import time, sleep
from typing import Any, Union
from typing import Any, Union, Dict, List
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
@ -44,11 +44,11 @@ class FileEventPattern(BasePattern):
# The variable name given to the triggering file within recipe code
triggering_file:str
# Which types of event the pattern responds to
event_mask:list[str]
event_mask:List[str]
def __init__(self, name:str, triggering_path:str, recipe:str,
triggering_file:str, event_mask:list[str]=_DEFAULT_MASK,
parameters:dict[str,Any]={}, outputs:dict[str,Any]={},
sweep:dict[str,Any]={}):
triggering_file:str, event_mask:List[str]=_DEFAULT_MASK,
parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={},
sweep:Dict[str,Any]={}):
"""FileEventPattern Constructor. This is used to match against file
system events, as caught by the python watchdog module."""
super().__init__(name, recipe, parameters, outputs, sweep)
@ -79,14 +79,14 @@ class FileEventPattern(BasePattern):
Called within parent BasePattern constructor."""
valid_string(recipe, VALID_RECIPE_NAME_CHARS)
def _is_valid_parameters(self, parameters:dict[str,Any])->None:
def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
"""Validation check for 'parameters' variable from main constructor.
Called within parent BasePattern constructor."""
valid_dict(parameters, str, Any, strict=False, min_length=0)
for k in parameters.keys():
valid_string(k, VALID_VARIABLE_NAME_CHARS)
def _is_valid_output(self, outputs:dict[str,str])->None:
def _is_valid_output(self, outputs:Dict[str,str])->None:
"""Validation check for 'output' variable from main constructor.
Called within parent BasePattern constructor."""
valid_dict(outputs, str, str, strict=False, min_length=0)
@ -101,7 +101,7 @@ class FileEventPattern(BasePattern):
raise ValueError(f"Invalid event mask '{mask}'. Valid are: "
f"{FILE_EVENTS}")
def _is_valid_sweep(self, sweep: dict[str,Union[int,float,complex]]) -> None:
def _is_valid_sweep(self, sweep: Dict[str,Union[int,float,complex]]) -> None:
"""Validation check for 'sweep' variable from main constructor."""
return super()._is_valid_sweep(sweep)
@ -124,8 +124,8 @@ class WatchdogMonitor(BaseMonitor):
#A lock to solve race conditions on '_rules'
_rules_lock:threading.Lock
def __init__(self, base_dir:str, patterns:dict[str,FileEventPattern],
recipes:dict[str,BaseRecipe], autostart=False, settletime:int=1,
def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern],
recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1,
print:Any=sys.stdout, logging:int=0)->None:
"""WatchdogEventHandler Constructor. This uses the watchdog module to
monitor a directory and all its sub-directories. Watchdog will provide
@ -189,7 +189,8 @@ class WatchdogMonitor(BaseMonitor):
# Use regex to match event paths against rule paths
target_path = rule.pattern.triggering_path
recursive_regexp = translate(target_path)
direct_regexp = recursive_regexp.replace('.*', '[^/]*')
direct_regexp = recursive_regexp.replace(
'.*', '[^'+ os.path.sep +']*')
recursive_hit = match(recursive_regexp, handle_path)
direct_hit = match(direct_regexp, handle_path)
@ -261,7 +262,7 @@ class WatchdogMonitor(BaseMonitor):
else:
self._identify_lost_rules(lost_pattern=pattern)
def get_patterns(self)->dict[str,FileEventPattern]:
def get_patterns(self)->Dict[str,FileEventPattern]:
"""Function to get a dict of the currently defined patterns of the
monitor. Note that the result is deep-copied, and so can be manipulated
without directly manipulating the internals of the monitor."""
@ -324,7 +325,7 @@ class WatchdogMonitor(BaseMonitor):
else:
self._identify_lost_rules(lost_recipe=recipe)
def get_recipes(self)->dict[str,BaseRecipe]:
def get_recipes(self)->Dict[str,BaseRecipe]:
"""Function to get a dict of the currently defined recipes of the
monitor. Note that the result is deep-copied, and so can be manipulated
without directly manipulating the internals of the monitor."""
@ -338,7 +339,7 @@ class WatchdogMonitor(BaseMonitor):
self._recipes_lock.release()
return to_return
def get_rules(self)->dict[str,BaseRule]:
def get_rules(self)->Dict[str,BaseRule]:
"""Function to get a dict of the currently defined rules of the
monitor. Note that the result is deep-copied, and so can be manipulated
without directly manipulating the internals of the monitor."""
@ -450,12 +451,12 @@ class WatchdogMonitor(BaseMonitor):
automatically called during initialisation."""
valid_existing_dir_path(base_dir)
def _is_valid_patterns(self, patterns:dict[str,FileEventPattern])->None:
def _is_valid_patterns(self, patterns:Dict[str,FileEventPattern])->None:
"""Validation check for 'patterns' variable from main constructor. Is
automatically called during initialisation."""
valid_dict(patterns, str, FileEventPattern, min_length=0, strict=False)
def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None:
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
"""Validation check for 'recipes' variable from main constructor. Is
automatically called during initialisation."""
valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False)
@ -511,8 +512,8 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
# A time to wait per event path, during which extra events are discared
_settletime:int
# TODO clean this struct occasionally
# A dict of recent job timestamps
_recent_jobs:dict[str, Any]
# A Dict of recent job timestamps
_recent_jobs:Dict[str, Any]
# A lock to solve race conditions on '_recent_jobs'
_recent_jobs_lock:threading.Lock
def __init__(self, monitor:WatchdogMonitor, settletime:int=1):

20
recipes/jupyter_notebook_recipe.py
View File

@ -9,7 +9,7 @@ import os
import nbformat
import sys
from typing import Any, Tuple
from typing import Any, Tuple, Dict
from core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_existing_dir_path, setup_debugging, \
@ -28,8 +28,8 @@ from core.meow import BaseRecipe, BaseHandler
class JupyterNotebookRecipe(BaseRecipe):
# A path to the jupyter notebook used to create this recipe
source:str
def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={},
requirements:dict[str,Any]={}, source:str=""):
def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={},
requirements:Dict[str,Any]={}, source:str=""):
"""JupyterNotebookRecipe Constructor. This is used to execute analysis
code using the papermill module."""
super().__init__(name, recipe, parameters, requirements)
@ -41,20 +41,20 @@ class JupyterNotebookRecipe(BaseRecipe):
if source:
valid_path(source, extension=".ipynb", min_length=0)
def _is_valid_recipe(self, recipe:dict[str,Any])->None:
def _is_valid_recipe(self, recipe:Dict[str,Any])->None:
"""Validation check for 'recipe' variable from main constructor.
Called within parent BaseRecipe constructor."""
check_type(recipe, dict)
check_type(recipe, Dict)
nbformat.validate(recipe)
def _is_valid_parameters(self, parameters:dict[str,Any])->None:
def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
"""Validation check for 'parameters' variable from main constructor.
Called within parent BaseRecipe constructor."""
valid_dict(parameters, str, Any, strict=False, min_length=0)
for k in parameters.keys():
valid_string(k, VALID_VARIABLE_NAME_CHARS)
def _is_valid_requirements(self, requirements:dict[str,Any])->None:
def _is_valid_requirements(self, requirements:Dict[str,Any])->None:
"""Validation check for 'requirements' variable from main constructor.
Called within parent BaseRecipe constructor."""
valid_dict(requirements, str, Any, strict=False, min_length=0)
@ -86,7 +86,7 @@ class PapermillHandler(BaseHandler):
print_debug(self._print_target, self.debug_level,
"Created new PapermillHandler instance", DEBUG_INFO)
def handle(self, event:dict[str,Any])->None:
def handle(self, event:Dict[str,Any])->None:
"""Function called to handle a given event."""
print_debug(self._print_target, self.debug_level,
f"Handling event {event[EVENT_PATH]}", DEBUG_INFO)
@ -112,7 +112,7 @@ class PapermillHandler(BaseHandler):
yaml_dict[value[0]] = value[1]
self.setup_job(event, yaml_dict)
def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]:
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. This handler accepts events from watchdog with
jupyter notebook recipes."""
@ -135,7 +135,7 @@ class PapermillHandler(BaseHandler):
constructor."""
valid_existing_dir_path(output_dir, allow_base=True)
def setup_job(self, event:dict[str,Any], yaml_dict:dict[str,Any])->None:
def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None:
"""Function to set up new job dict and send it to the runner to be
executed."""
meow_job = create_job(

20
recipes/python_recipe.py
View File

@ -8,7 +8,7 @@ Author(s): David Marchant
import os
import sys
from typing import Any, Tuple
from typing import Any, Tuple, Dict, List
from core.correctness.validation import check_script, valid_string, \
valid_dict, valid_event, valid_existing_dir_path, setup_debugging
@ -24,31 +24,32 @@ from core.meow import BaseRecipe, BaseHandler
class PythonRecipe(BaseRecipe):
def __init__(self, name:str, recipe:list[str], parameters:dict[str,Any]={},
requirements:dict[str,Any]={}):
def __init__(self, name:str, recipe:List[str], parameters:Dict[str,Any]={},
requirements:Dict[str,Any]={}):
"""PythonRecipe Constructor. This is used to execute python analysis
code."""
super().__init__(name, recipe, parameters, requirements)
def _is_valid_recipe(self, recipe:list[str])->None:
def _is_valid_recipe(self, recipe:List[str])->None:
"""Validation check for 'recipe' variable from main constructor.
Called within parent BaseRecipe constructor."""
check_script(recipe)
def _is_valid_parameters(self, parameters:dict[str,Any])->None:
def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
"""Validation check for 'parameters' variable from main constructor.
Called within parent BaseRecipe constructor."""
valid_dict(parameters, str, Any, strict=False, min_length=0)
for k in parameters.keys():
valid_string(k, VALID_VARIABLE_NAME_CHARS)
def _is_valid_requirements(self, requirements:dict[str,Any])->None:
def _is_valid_requirements(self, requirements:Dict[str,Any])->None:
"""Validation check for 'requirements' variable from main constructor.
Called within parent BaseRecipe constructor."""
valid_dict(requirements, str, Any, strict=False, min_length=0)
for k in requirements.keys():
valid_string(k, VALID_VARIABLE_NAME_CHARS)
class PythonHandler(BaseHandler):
# TODO move me to base handler
# handler directory to setup jobs in
@ -75,7 +76,7 @@ class PythonHandler(BaseHandler):
print_debug(self._print_target, self.debug_level,
"Created new PythonHandler instance", DEBUG_INFO)
def handle(self, event:dict[str,Any])->None:
def handle(self, event:Dict[str,Any])->None:
"""Function called to handle a given event."""
print_debug(self._print_target, self.debug_level,
f"Handling event {event[EVENT_PATH]}", DEBUG_INFO)
@ -101,7 +102,7 @@ class PythonHandler(BaseHandler):
yaml_dict[value[0]] = value[1]
self.setup_job(event, yaml_dict)
def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]:
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. This handler accepts events from watchdog with
Python recipes"""
@ -124,7 +125,7 @@ class PythonHandler(BaseHandler):
constructor."""
valid_existing_dir_path(output_dir, allow_base=True)
def setup_job(self, event:dict[str,Any], yaml_dict:dict[str,Any])->None:
def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None:
"""Function to set up new job dict and send it to the runner to be
executed."""
meow_job = create_job(
@ -175,6 +176,7 @@ class PythonHandler(BaseHandler):
# Send job directory, as actual definitons will be read from within it
self.to_runner.send(job_dir)
# Papermill job execution code, to be run within the conductor
def python_job_func(job):
# Requires own imports as will be run in its own execution environment

14
tests/shared.py
View File

@ -3,6 +3,8 @@ This file contains shared functions and variables used within multiple tests.
Author(s): David Marchant
"""
import os
from core.functionality import make_dir, rmtree
@ -30,8 +32,8 @@ COMPLETE_PYTHON_SCRIPT = [
"import os",
"# Setup parameters",
"num = 1000",
"infile = 'somehere/particular'",
"outfile = 'nowhere/particular'",
"infile = 'somehere"+ os.path.sep +"particular'",
"outfile = 'nowhere"+ os.path.sep +"particular'",
"",
"with open(infile, 'r') as file:",
" s = float(file.read())",
@ -131,9 +133,9 @@ APPENDING_NOTEBOOK = {
"# The line to append\n",
"extra = 'This line comes from a default pattern'\n",
"# Data input file location\n",
"infile = 'start/alpha.txt'\n",
"infile = 'start"+ os.path.sep +"alpha.txt'\n",
"# Output file location\n",
"outfile = 'first/alpha.txt'"
"outfile = 'first"+ os.path.sep +"alpha.txt'"
]
},
{
@ -216,9 +218,9 @@ ADDING_NOTEBOOK = {
"# Amount to add to data\n",
"extra = 10\n",
"# Data input file location\n",
"infile = 'example_data/data_0.npy'\n",
"infile = 'example_data"+ os.path.sep +"data_0.npy'\n",
"# Output file location\n",
"outfile = 'standard_output/data_0.npy'"
"outfile = 'standard_output"+ os.path.sep +"data_0.npy'"
]
},
{

6
tests/test_conductors.py
View File

@ -2,6 +2,8 @@
import os
import unittest
from typing import Dict
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
@ -108,7 +110,7 @@ class MeowTests(unittest.TestCase):
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
@ -196,7 +198,7 @@ class MeowTests(unittest.TestCase):
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)

77
tests/test_functionality.py
View File

@ -383,28 +383,38 @@ class CorrectnessTests(unittest.TestCase):
}
replaced = replace_keywords(
test_dict, "job_id", "base/src/dir/file.ext", "base/monitor/dir")
test_dict,
"job_id",
os.path.join("base", "src", "dir", "file.ext"),
os.path.join("base", "monitor", "dir")
)
self.assertIsInstance(replaced, dict)
self.assertEqual(len(test_dict.keys()), len(replaced.keys()))
for k in test_dict.keys():
self.assertIn(k, replaced)
self.assertEqual(replaced["A"], "--base/src/dir/file.ext--")
self.assertEqual(replaced["B"], "--../../src/dir/file.ext--")
self.assertEqual(replaced["C"], "--base/src/dir--")
self.assertEqual(replaced["D"], "--../../src/dir--")
self.assertEqual(replaced["A"],
os.path.join("--base", "src", "dir", "file.ext--"))
self.assertEqual(replaced["B"],
os.path.join("--..", "..", "src", "dir", "file.ext--"))
self.assertEqual(replaced["C"],
os.path.join("--base", "src", "dir--"))
self.assertEqual(replaced["D"],
os.path.join("--..", "..", "src", "dir--"))
self.assertEqual(replaced["E"], "--file.ext--")
self.assertEqual(replaced["F"], "--file--")
self.assertEqual(replaced["G"], "--base/monitor/dir--")
self.assertEqual(replaced["G"],
os.path.join("--base", "monitor", "dir--"))
self.assertEqual(replaced["H"], "--.ext--")
self.assertEqual(replaced["I"], "--job_id--")
self.assertEqual(replaced["J"],
"--base/src/dir/file.ext-base/src/dir/file.ext--")
self.assertEqual(replaced["K"], "base/src/dir/file.ext")
os.path.join("--base", "src", "dir", "file.ext-base", "src", "dir", "file.ext--"))
self.assertEqual(replaced["K"],
os.path.join("base", "src", "dir", "file.ext"))
self.assertEqual(replaced["L"],
"--base/src/dir/file.ext-../../src/dir/file.ext-base/src/dir-"
"../../src/dir-file.ext-file-base/monitor/dir-.ext-job_id--")
os.path.join("--base", "src", "dir", "file.ext-..", "..", "src", "dir", "file.ext-base", "src", "dir-"
"..", "..", "src", "dir-file.ext-file-base", "monitor", "dir-.ext-job_id--"))
self.assertEqual(replaced["M"], "A")
self.assertEqual(replaced["N"], 1)
@ -498,30 +508,31 @@ data"""
'"metadata": {}, "outputs": [], "source": ["# Default parameters '
'values\\n", "# The line to append\\n", "extra = \'This line '
'comes from a default pattern\'\\n", "# Data input file '
'location\\n", "infile = \'start/alpha.txt\'\\n", "# Output file '
'location\\n", "outfile = \'first/alpha.txt\'"]}, {"cell_type": '
'location\\n", "infile = \'start'+ os.path.sep +'alpha.txt\'\\n", '
'"# Output file location\\n", "outfile = \'first'+ os.path.sep
+'alpha.txt\'"]}, {"cell_type": "code", "execution_count": null, '
'"metadata": {}, "outputs": [], "source": ["# load in dataset. '
'This should be a text file\\n", "with open(infile) as '
'input_file:\\n", " data = input_file.read()"]}, {"cell_type": '
'"code", "execution_count": null, "metadata": {}, "outputs": [], '
'"source": ["# load in dataset. This should be a text file\\n", '
'"with open(infile) as input_file:\\n", " data = '
'input_file.read()"]}, {"cell_type": "code", "execution_count": '
'null, "metadata": {}, "outputs": [], "source": ["# Append the '
'line\\n", "appended = data + \'\\\\n\' + extra"]}, {"cell_type": '
'"code", "execution_count": null, "metadata": {}, "outputs": [], '
'"source": ["import os\\n", "\\n", "# Create output directory if '
'it doesn\'t exist\\n", "output_dir_path = '
'os.path.dirname(outfile)\\n", "\\n", "if output_dir_path:\\n", '
'" os.makedirs(output_dir_path, exist_ok=True)\\n", "\\n", "# '
'Save added array as new dataset\\n", "with open(outfile, \'w\') '
'as output_file:\\n", " output_file.write(appended)"]}], '
'"metadata": {"kernelspec": {"display_name": "Python 3", '
'"language": "python", "name": "python3"}, "language_info": '
'{"codemirror_mode": {"name": "ipython", "version": 3}, '
'"file_extension": ".py", "mimetype": "text/x-python", "name": '
'"python", "nbconvert_exporter": "python", "pygments_lexer": '
'"ipython3", "version": "3.10.6 (main, Nov 14 2022, 16:10:14) '
'[GCC 11.3.0]"}, "vscode": {"interpreter": {"hash": '
'"916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1'
'"}}}, "nbformat": 4, "nbformat_minor": 4}'
'"source": ["# Append the line\\n", "appended = data + \'\\\\n\' '
'+ extra"]}, {"cell_type": "code", "execution_count": null, '
'"metadata": {}, "outputs": [], "source": ["import os\\n", "\\n", '
'"# Create output directory if it doesn\'t exist\\n", '
'"output_dir_path = os.path.dirname(outfile)\\n", "\\n", '
'"if output_dir_path:\\n", " os.makedirs(output_dir_path, '
'exist_ok=True)\\n", "\\n", "# Save added array as new '
'dataset\\n", "with open(outfile, \'w\') as output_file:\\n", " '
'output_file.write(appended)"]}], "metadata": {"kernelspec": '
'{"display_name": "Python 3", "language": "python", "name": '
'"python3"}, "language_info": {"codemirror_mode": {"name": '
'"ipython", "version": 3}, "file_extension": ".py", "mimetype": '
'"text/x-python", "name": "python", "nbconvert_exporter": '
'"python", "pygments_lexer": "ipython3", "version": "3.10.6 '
'(main, Nov 14 2022, 16:10:14) [GCC 11.3.0]"}, "vscode": '
'{"interpreter": {"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c'
'65e1c03b4aa3499c5328201f1"}}}, "nbformat": 4, "nbformat_minor": '
'4}'
]
self.assertEqual(data, expected_bytes)

16
tests/test_meow.py
View File

@ -1,7 +1,7 @@
import unittest
from typing import Any, Union, Tuple
from typing import Any, Union, Tuple, Dict
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
@ -67,7 +67,7 @@ class MeowTests(unittest.TestCase):
def _is_valid_output(self, outputs:Any)->None:
pass
def _is_valid_sweep(self,
sweep:dict[str,Union[int,float,complex]])->None:
sweep:Dict[str,Union[int,float,complex]])->None:
pass
FullPattern("name", "", "", "", "")
@ -188,7 +188,7 @@ class MeowTests(unittest.TestCase):
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, dict)
self.assertIsInstance(rules, Dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
@ -229,9 +229,9 @@ class MeowTests(unittest.TestCase):
pass
def stop(self):
pass
def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None:
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
pass
def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None:
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
pass
def add_pattern(self, pattern:BasePattern)->None:
pass
@ -274,7 +274,7 @@ class MeowTests(unittest.TestCase):
pass
def _is_valid_inputs(self, inputs:Any)->None:
pass
def valid_handle_criteria(self, event:dict[str,Any]
def valid_handle_criteria(self, event:Dict[str,Any]
)->Tuple[bool,str]:
pass
@ -292,10 +292,10 @@ class MeowTests(unittest.TestCase):
TestConductor()
class FullTestConductor(BaseConductor):
def execute(self, job:dict[str,Any])->None:
def execute(self, job:Dict[str,Any])->None:
pass
def valid_execute_criteria(self, job:dict[str,Any]
def valid_execute_criteria(self, job:Dict[str,Any]
)->Tuple[bool,str]:
pass

60
tests/test_patterns.py
View File

@ -245,7 +245,10 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor identifies expected events in sub directories
def testMonitoring(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
@ -304,7 +307,10 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor identifies fake events for retroactive patterns
def testMonitoringRetroActive(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
@ -368,10 +374,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor get_patterns function
def testMonitorGetPatterns(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -395,10 +407,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor add_pattern function
def testMonitorAddPattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -440,10 +458,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor update_patterns function
def testMonitorUpdatePattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -502,10 +526,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor remove_patterns function
def testMonitorRemovePattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -707,10 +737,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor get_rules function
def testMonitorGetRules(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)

5
tests/test_recipes.py
View File

@ -4,6 +4,7 @@ import os
import unittest
from multiprocessing import Pipe
from typing import Dict
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
@ -391,7 +392,7 @@ class JupyterNotebookTests(unittest.TestCase):
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
@ -759,7 +760,7 @@ class PythonTests(unittest.TestCase):
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
self.assertNotIn(JOB_ERROR, status)

30
tests/test_runner.py
View File

@ -121,10 +121,13 @@ class MeowTests(unittest.TestCase):
# Test single meow papermill job execution
def testMeowRunnerPapermillExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
@ -205,16 +208,19 @@ class MeowTests(unittest.TestCase):
# Test meow papermill job chaining within runner
def testMeowRunnerLinkedPapermillExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={
"extra":"A line from Pattern 1",
"outfile":"{VGRID}/middle/{FILENAME}"
"outfile":os.path.join("{VGRID}", "middle", "{FILENAME}")
})
pattern_two = FileEventPattern(
"pattern_two", "middle/A.txt", "recipe_one", "infile",
"pattern_two", os.path.join("middle", "A.txt"), "recipe_one", "infile",
parameters={
"extra":"A line from Pattern 2",
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
@ -314,10 +320,10 @@ class MeowTests(unittest.TestCase):
# Test single meow python job execution
def testMeowRunnerPythonExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one", os.path.join("start", "A.txt"), "recipe_one", "infile",
parameters={
"num":10000,
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT
@ -403,16 +409,16 @@ class MeowTests(unittest.TestCase):
# Test meow python job chaining within runner
def testMeowRunnerLinkedPythonExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one", os.path.join("start", "A.txt"), "recipe_one", "infile",
parameters={
"num":250,
"outfile":"{VGRID}/middle/{FILENAME}"
"outfile":os.path.join("{VGRID}", "middle", "{FILENAME}")
})
pattern_two = FileEventPattern(
"pattern_two", "middle/A.txt", "recipe_one", "infile",
"pattern_two", os.path.join("middle", "A.txt"), "recipe_one", "infile",
parameters={
"num":40,
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT

5
tests/test_validation.py
View File

@ -239,9 +239,10 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(ValueError):
valid_non_existing_path("first")
make_dir("first/second")
test_path = os.path.join("first", "second")
make_dir(test_path)
with self.assertRaises(ValueError):
valid_non_existing_path("first/second")
valid_non_existing_path(test_path)
# TODO modify so tests for actual rule values
# Test valid_event can check given event dictionary