diff --git a/README.md b/README.md index fc48fe1..d2bd68e 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,8 @@ Pytest unittests are provided within the 'tests directory, as well as a script * with individual tests runnable with: pytest test_runner.py::MeowTests::testMeowRunnerLinkedPythonExecution -W ignore::DeprecationWarning + +to run locally, update your '~/.bashrc' file to inclue: + + # Manually added to get local testing of Python files working easier + export PYTHONPATH=/home/patch/Documents/Research/Python \ No newline at end of file diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index 34522a0..4586e82 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -9,7 +9,7 @@ import os import shutil from datetime import datetime -from typing import Any, Tuple +from typing import Any, Tuple, Dict from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \ @@ -25,7 +25,7 @@ class LocalPythonConductor(BaseConductor): def __init__(self)->None: super().__init__() - def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]: + def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an job defintion, if this conductor can process it or not. This conductor will accept any Python job type""" try: @@ -36,7 +36,7 @@ class LocalPythonConductor(BaseConductor): pass return False, str(e) - def execute(self, job:dict[str,Any])->None: + def execute(self, job:Dict[str,Any])->None: valid_job(job) job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID]) diff --git a/core/correctness/validation.py b/core/correctness/validation.py index 86f62b7..4e9c752 100644 --- a/core/correctness/validation.py +++ b/core/correctness/validation.py @@ -8,7 +8,8 @@ Author(s): David Marchant from datetime import datetime from inspect import signature from os.path import sep, exists, isfile, isdir, dirname -from typing import Any, _SpecialForm, Union, Tuple, get_origin, get_args +from typing import Any, _SpecialForm, Union, Tuple, Type, Dict, List, \ + get_origin, get_args from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \ EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \ @@ -33,7 +34,7 @@ WATCHDOG_EVENT_KEYS = { # Required keys in job dict JOB_KEYS = { JOB_TYPE: str, - JOB_EVENT: dict, + JOB_EVENT: Dict, JOB_ID: str, JOB_PATTERN: Any, JOB_RECIPE: Any, @@ -42,7 +43,7 @@ JOB_KEYS = { JOB_CREATE_TIME: datetime, } -def check_type(variable:Any, expected_type:type, alt_types:list[type]=[], +def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[], or_none:bool=False)->None: """Checks if a given variable is of the expected type. Raises TypeError or ValueError as appropriate if any issues are encountered.""" @@ -52,6 +53,11 @@ def check_type(variable:Any, expected_type:type, alt_types:list[type]=[], if get_origin(expected_type) is Union: type_list = list(get_args(expected_type)) type_list = type_list + alt_types +# # If we have any types from typing, then update to allow checks against +# # their base types too +# for t in type_list: +# if get_origin(t): +# type_list.append(get_origin(t)) # Only accept None if explicitly allowed if variable is None: @@ -130,16 +136,16 @@ def valid_string(variable:str, valid_chars:str, min_length:int=1)->None: "%s" % (char, valid_chars) ) -def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type, - required_keys:list[Any]=[], optional_keys:list[Any]=[], +def valid_dict(variable:Dict[Any, Any], key_type:Type, value_type:Type, + required_keys:List[Any]=[], optional_keys:List[Any]=[], strict:bool=True, min_length:int=1)->None: """Checks that a given dictionary is valid. Key and Value types are enforced, as are required and optional keys. Will raise ValueError, TypeError or KeyError depending on the problem encountered.""" # Validate inputs - check_type(variable, dict) - check_type(key_type, type, alt_types=[_SpecialForm]) - check_type(value_type, type, alt_types=[_SpecialForm]) + check_type(variable, Dict) + check_type(key_type, Type, alt_types=[_SpecialForm]) + check_type(value_type, Type, alt_types=[_SpecialForm]) check_type(required_keys, list) check_type(optional_keys, list) check_type(strict, bool) @@ -172,11 +178,11 @@ def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type, raise ValueError(f"Unexpected key '{k}' should not be present " f"in dict '{variable}'") -def valid_list(variable:list[Any], entry_type:type, - alt_types:list[type]=[], min_length:int=1)->None: +def valid_list(variable:List[Any], entry_type:Type, + alt_types:List[Type]=[], min_length:int=1)->None: """Checks that a given list is valid. Value types are checked and a ValueError or TypeError is raised if a problem is encountered.""" - check_type(variable, list) + check_type(variable, List) # Check length meets minimum if len(variable) < min_length: @@ -256,11 +262,11 @@ def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]: return print, logging -def valid_meow_dict(meow_dict:dict[str,Any], msg:str, - keys:dict[str,type])->None: +def valid_meow_dict(meow_dict:Dict[str,Any], msg:str, + keys:Dict[str,Type])->None: """Check given dictionary expresses a meow construct. This won't do much directly, but is called by more specific validation functions.""" - check_type(meow_dict, dict) + check_type(meow_dict, Dict) # Check we have all the required keys, and they are all of the expected # type for key, value_type in keys.items(): @@ -268,13 +274,13 @@ def valid_meow_dict(meow_dict:dict[str,Any], msg:str, raise KeyError(f"{msg} require key '{key}'") check_type(meow_dict[key], value_type) -def valid_event(event:dict[str,Any])->None: +def valid_event(event:Dict[str,Any])->None: """Check that a given dict expresses a meow event.""" valid_meow_dict(event, "Event", EVENT_KEYS) -def valid_job(job:dict[str,Any])->None: +def valid_job(job:Dict[str,Any])->None: """Check that a given dict expresses a meow job.""" valid_meow_dict(job, "Job", JOB_KEYS) -def valid_watchdog_event(event:dict[str,Any])->None: +def valid_watchdog_event(event:Dict[str,Any])->None: valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS) diff --git a/core/functionality.py b/core/functionality.py index 98af242..3e1c204 100644 --- a/core/functionality.py +++ b/core/functionality.py @@ -7,11 +7,12 @@ import os import yaml from datetime import datetime +from typing import List from multiprocessing.connection import Connection, wait as multi_wait from multiprocessing.queues import Queue from papermill.translators import papermill_translators -from typing import Any +from typing import Any, Dict from random import SystemRandom from core.correctness.validation import check_type, valid_existing_file_path, \ @@ -36,7 +37,7 @@ KEYWORD_JOB = "{JOB}" #TODO Make this guaranteed unique -def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[], +def generate_id(prefix:str="", length:int=16, existing_ids:List[str]=[], charset:str=CHAR_UPPERCASE+CHAR_LOWERCASE, attempts:int=24): random_length = max(length - len(prefix), 0) for _ in range(attempts): @@ -47,10 +48,12 @@ def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[], raise ValueError(f"Could not generate ID unique from '{existing_ids}' " f"using values '{charset}' and length of '{length}'.") -def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]: +def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]: all_connections = [i for i in inputs if type(i) is Connection] \ + [i._reader for i in inputs if type(i) is Queue] - + for i in inputs: + print(type(i)) + ready = multi_wait(all_connections) ready_inputs = [i for i in inputs if \ (type(i) is Connection and i in ready) \ @@ -72,7 +75,6 @@ def _get_file_sha256(file_path): def get_file_hash(file_path:str, hash:str): check_type(hash, str) - import os valid_existing_file_path(file_path) valid_hashes = { @@ -80,7 +82,7 @@ def get_file_hash(file_path:str, hash:str): } if hash not in valid_hashes: raise KeyError(f"Cannot use hash '{hash}'. Valid are " - "'{list(valid_hashes.keys())}") + f"'{list(valid_hashes.keys())}") return valid_hashes[hash](file_path) @@ -169,7 +171,7 @@ def read_notebook(filepath:str): with open(filepath, 'r') as read_file: return json.load(read_file) -def write_notebook(source:dict[str,Any], filename:str): +def write_notebook(source:Dict[str,Any], filename:str): """ Writes the given notebook source code to a given filename. @@ -183,10 +185,10 @@ def write_notebook(source:dict[str,Any], filename:str): json.dump(source, job_file) # Adapted from: https://github.com/rasmunk/notebook_parameterizer -def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any], - parameters:dict[str,Any], expand_env_values:bool=False)->dict[str,Any]: +def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any], + parameters:Dict[str,Any], expand_env_values:bool=False)->Dict[str,Any]: nbformat.validate(jupyter_notebook) - check_type(parameters, dict) + check_type(parameters, Dict) if jupyter_notebook["nbformat"] != 4: raise Warning( @@ -256,10 +258,10 @@ def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any], return output_notebook -def parameterize_python_script(script:list[str], parameters:dict[str,Any], - expand_env_values:bool=False)->dict[str,Any]: +def parameterize_python_script(script:List[str], parameters:Dict[str,Any], + expand_env_values:bool=False)->Dict[str,Any]: check_script(script) - check_type(parameters, dict) + check_type(parameters, Dict) output_script = copy.deepcopy(script) @@ -302,8 +304,8 @@ def print_debug(print_target, debug_level, msg, level)->None: status = "WARNING" print(f"{status}: {msg}", file=print_target) -def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str, - monitor_base:str)->dict[str,str]: +def replace_keywords(old_dict:Dict[str,str], job_id:str, src_path:str, + monitor_base:str)->Dict[str,str]: """Function to replace all MEOW magic words in a dictionary with dynamic values.""" new_dict = {} @@ -332,8 +334,8 @@ def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str, return new_dict -def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={} - )->dict[Any,Any]: +def create_event(event_type:str, path:str, rule:Any, extras:Dict[Any,Any]={} + )->Dict[Any,Any]: """Function to create a MEOW dictionary.""" return { **extras, @@ -343,7 +345,7 @@ def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={} } def create_watchdog_event(path:str, rule:Any, base:str, hash:str, - extras:dict[Any,Any]={})->dict[Any,Any]: + extras:Dict[Any,Any]={})->Dict[Any,Any]: """Function to create a MEOW event dictionary.""" return create_event( EVENT_TYPE_WATCHDOG, @@ -358,8 +360,8 @@ def create_watchdog_event(path:str, rule:Any, base:str, hash:str, } ) -def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={} - )->dict[Any,Any]: +def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={} + )->Dict[Any,Any]: """Function to create a MEOW job dictionary.""" job_dict = { #TODO compress event? @@ -376,7 +378,7 @@ def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={} return {**extras, **job_dict} -def lines_to_string(lines:list[str])->str: +def lines_to_string(lines:List[str])->str: """Function to convert a list of str lines, into one continuous string separated by newline characters""" return "\n".join(lines) diff --git a/core/meow.py b/core/meow.py index b4287e9..63371be 100644 --- a/core/meow.py +++ b/core/meow.py @@ -12,7 +12,7 @@ import itertools import sys from copy import deepcopy -from typing import Any, Union, Tuple +from typing import Any, Union, Tuple, Dict, List from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \ @@ -28,11 +28,11 @@ class BaseRecipe: # Actual code to run recipe:Any # Possible parameters that could be overridden by a Pattern - parameters:dict[str, Any] + parameters:Dict[str, Any] # Additional configuration options - requirements:dict[str, Any] - def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={}, - requirements:dict[str,Any]={}): + requirements:Dict[str, Any] + def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, + requirements:Dict[str,Any]={}): """BaseRecipe Constructor. This will check that any class inheriting from it implements its validation functions. It will then call these on the input parameters.""" @@ -84,14 +84,14 @@ class BasePattern: # An identifier of a recipe recipe:str # Parameters to be overridden in the recipe - parameters:dict[str,Any] + parameters:Dict[str,Any] # Parameters showing the potential outputs of a recipe - outputs:dict[str,Any] + outputs:Dict[str,Any] # A collection of variables to be swept over for job scheduling - sweep:dict[str,Any] + sweep:Dict[str,Any] - def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={}, - outputs:dict[str,Any]={}, sweep:dict[str,Any]={}): + def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={}, + outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}): """BasePattern Constructor. This will check that any class inheriting from it implements its validation functions. It will then call these on the input parameters.""" @@ -138,11 +138,11 @@ class BasePattern: be implemented by any child class.""" pass - def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None: + def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None: """Validation check for 'sweep' variable from main constructor. This function is implemented to check for the types given in the signature, and must be overridden if these differ.""" - check_type(sweep, dict) + check_type(sweep, Dict) if not sweep: return for _, v in sweep.items(): @@ -175,7 +175,7 @@ class BasePattern: "value where the end point is smaller than the start." ) - def expand_sweeps(self)->list[Tuple[str,Any]]: + def expand_sweeps(self)->List[Tuple[str,Any]]: """Function to get all combinations of sweep parameters""" values_dict = {} # get a collection of a individual sweep values @@ -259,17 +259,17 @@ class BaseRule: class BaseMonitor: # A collection of patterns - _patterns: dict[str, BasePattern] + _patterns: Dict[str, BasePattern] # A collection of recipes - _recipes: dict[str, BaseRecipe] + _recipes: Dict[str, BaseRecipe] # A collection of rules derived from _patterns and _recipes - _rules: dict[str, BaseRule] + _rules: Dict[str, BaseRule] # A channel for sending messages to the runner. Note that this is not # initialised within the constructor, but within the runner when passed the # monitor is passed to it. to_runner: VALID_CHANNELS - def __init__(self, patterns:dict[str,BasePattern], - recipes:dict[str,BaseRecipe])->None: + def __init__(self, patterns:Dict[str,BasePattern], + recipes:Dict[str,BaseRecipe])->None: """BaseMonitor Constructor. This will check that any class inheriting from it implements its validation functions. It will then call these on the input parameters.""" @@ -302,12 +302,12 @@ class BaseMonitor: raise TypeError(msg) return object.__new__(cls) - def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None: + def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None: """Validation check for 'patterns' variable from main constructor. Must be implemented by any child class.""" pass - def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None: + def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None: """Validation check for 'recipes' variable from main constructor. Must be implemented by any child class.""" pass @@ -337,7 +337,7 @@ class BaseMonitor: implemented by any child process.""" pass - def get_patterns(self)->dict[str,BasePattern]: + def get_patterns(self)->Dict[str,BasePattern]: """Function to get a dictionary of all current pattern definitions. Must be implemented by any child process.""" pass @@ -357,12 +357,12 @@ class BaseMonitor: implemented by any child process.""" pass - def get_recipes(self)->dict[str,BaseRecipe]: + def get_recipes(self)->Dict[str,BaseRecipe]: """Function to get a dictionary of all current recipe definitions. Must be implemented by any child process.""" pass - def get_rules(self)->dict[str,BaseRule]: + def get_rules(self)->Dict[str,BaseRule]: """Function to get a dictionary of all current rule definitions. Must be implemented by any child process.""" pass @@ -387,12 +387,12 @@ class BaseHandler: raise TypeError(msg) return object.__new__(cls) - def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]: + def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an event defintion, if this handler can process it or not. Must be implemented by any child process.""" pass - def handle(self, event:dict[str,Any])->None: + def handle(self, event:Dict[str,Any])->None: """Function to handle a given event. Must be implemented by any child process.""" pass @@ -413,28 +413,28 @@ class BaseConductor: raise TypeError(msg) return object.__new__(cls) - def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]: + def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an job defintion, if this conductor can process it or not. Must be implemented by any child process.""" pass - def execute(self, job:dict[str,Any])->None: + def execute(self, job:Dict[str,Any])->None: """Function to execute a given job. Must be implemented by any child process.""" pass -def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]], - recipes:Union[dict[str,BaseRecipe],list[BaseRecipe]], - new_rules:list[BaseRule]=[])->dict[str,BaseRule]: +def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], + recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]], + new_rules:List[BaseRule]=[])->Dict[str,BaseRule]: """Function to create any valid rules from a given collection of patterns and recipes. All inbuilt rule types are considered, with additional definitions provided through the 'new_rules' variable. Note that any provided pattern and recipe dictionaries must be keyed with the corresponding pattern and recipe names.""" # Validation of inputs - check_type(patterns, dict, alt_types=[list]) - check_type(recipes, dict, alt_types=[list]) + check_type(patterns, Dict, alt_types=[List]) + check_type(recipes, Dict, alt_types=[List]) valid_list(new_rules, BaseRule, min_length=0) # Convert a pattern list to a dictionary @@ -477,7 +477,7 @@ def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]], return generated_rules def create_rule(pattern:BasePattern, recipe:BaseRecipe, - new_rules:list[BaseRule]=[])->BaseRule: + new_rules:List[BaseRule]=[])->BaseRule: """Function to create a valid rule from a given pattern and recipe. All inbuilt rule types are considered, with additional definitions provided through the 'new_rules' variable.""" diff --git a/core/runner.py b/core/runner.py index ac5c9f9..6143e4e 100644 --- a/core/runner.py +++ b/core/runner.py @@ -12,7 +12,7 @@ import threading from multiprocessing import Pipe from random import randrange -from typing import Any, Union +from typing import Any, Union, Dict, List from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \ VALID_CHANNELS, JOB_ID, META_FILE @@ -24,18 +24,18 @@ from core.meow import BaseHandler, BaseMonitor, BaseConductor class MeowRunner: # A collection of all monitors in the runner - monitors:list[BaseMonitor] + monitors:List[BaseMonitor] # A collection of all handlers in the runner - handlers:list[BaseHandler] + handlers:List[BaseHandler] # A collection of all conductors in the runner - conductors:list[BaseConductor] + conductors:List[BaseConductor] # A collection of all channels from each monitor - from_monitors: list[VALID_CHANNELS] + from_monitors: List[VALID_CHANNELS] # A collection of all channels from each handler - from_handlers: list[VALID_CHANNELS] - def __init__(self, monitors:Union[BaseMonitor,list[BaseMonitor]], - handlers:Union[BaseHandler,list[BaseHandler]], - conductors:Union[BaseConductor,list[BaseConductor]], + from_handlers: List[VALID_CHANNELS] + def __init__(self, monitors:Union[BaseMonitor,List[BaseMonitor]], + handlers:Union[BaseHandler,List[BaseHandler]], + conductors:Union[BaseConductor,List[BaseConductor]], print:Any=sys.stdout, logging:int=0)->None: """MeowRunner constructor. This connects all provided monitors, handlers and conductors according to what events and jobs they produce @@ -178,7 +178,7 @@ class MeowRunner: ] self.execute_job(conductor, job) - def handle_event(self, handler:BaseHandler, event:dict[str:Any])->None: + def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None: """Function for a given handler to handle a given event, without crashing the runner in the event of a problem.""" print_debug(self._print_target, self.debug_level, @@ -193,7 +193,7 @@ class MeowRunner: "Something went wrong during handling for event " f"'{event[EVENT_TYPE]}'. {e}", DEBUG_INFO) - def execute_job(self, conductor:BaseConductor, job:dict[str:Any])->None: + def execute_job(self, conductor:BaseConductor, job:Dict[str,Any])->None: """Function for a given conductor to execute a given job, without crashing the runner in the event of a problem.""" print_debug(self._print_target, self.debug_level, @@ -306,22 +306,22 @@ class MeowRunner: "Job conductor thread stopped", DEBUG_INFO) def _is_valid_monitors(self, - monitors:Union[BaseMonitor,list[BaseMonitor]])->None: + monitors:Union[BaseMonitor,List[BaseMonitor]])->None: """Validation check for 'monitors' variable from main constructor.""" - check_type(monitors, BaseMonitor, alt_types=[list]) + check_type(monitors, BaseMonitor, alt_types=[List]) if type(monitors) == list: valid_list(monitors, BaseMonitor, min_length=1) def _is_valid_handlers(self, - handlers:Union[BaseHandler,list[BaseHandler]])->None: + handlers:Union[BaseHandler,List[BaseHandler]])->None: """Validation check for 'handlers' variable from main constructor.""" - check_type(handlers, BaseHandler, alt_types=[list]) + check_type(handlers, BaseHandler, alt_types=[List]) if type(handlers) == list: valid_list(handlers, BaseHandler, min_length=1) def _is_valid_conductors(self, - conductors:Union[BaseConductor,list[BaseConductor]])->None: + conductors:Union[BaseConductor,List[BaseConductor]])->None: """Validation check for 'conductors' variable from main constructor.""" - check_type(conductors, BaseConductor, alt_types=[list]) + check_type(conductors, BaseConductor, alt_types=[List]) if type(conductors) == list: valid_list(conductors, BaseConductor, min_length=1) diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 73ff89f..e2d9f9b 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -14,7 +14,7 @@ from copy import deepcopy from fnmatch import translate from re import match from time import time, sleep -from typing import Any, Union +from typing import Any, Union, Dict, List from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler @@ -44,11 +44,11 @@ class FileEventPattern(BasePattern): # The variable name given to the triggering file within recipe code triggering_file:str # Which types of event the pattern responds to - event_mask:list[str] + event_mask:List[str] def __init__(self, name:str, triggering_path:str, recipe:str, - triggering_file:str, event_mask:list[str]=_DEFAULT_MASK, - parameters:dict[str,Any]={}, outputs:dict[str,Any]={}, - sweep:dict[str,Any]={}): + triggering_file:str, event_mask:List[str]=_DEFAULT_MASK, + parameters:Dict[str,Any]={}, outputs:Dict[str,Any]={}, + sweep:Dict[str,Any]={}): """FileEventPattern Constructor. This is used to match against file system events, as caught by the python watchdog module.""" super().__init__(name, recipe, parameters, outputs, sweep) @@ -79,14 +79,14 @@ class FileEventPattern(BasePattern): Called within parent BasePattern constructor.""" valid_string(recipe, VALID_RECIPE_NAME_CHARS) - def _is_valid_parameters(self, parameters:dict[str,Any])->None: + def _is_valid_parameters(self, parameters:Dict[str,Any])->None: """Validation check for 'parameters' variable from main constructor. Called within parent BasePattern constructor.""" valid_dict(parameters, str, Any, strict=False, min_length=0) for k in parameters.keys(): valid_string(k, VALID_VARIABLE_NAME_CHARS) - def _is_valid_output(self, outputs:dict[str,str])->None: + def _is_valid_output(self, outputs:Dict[str,str])->None: """Validation check for 'output' variable from main constructor. Called within parent BasePattern constructor.""" valid_dict(outputs, str, str, strict=False, min_length=0) @@ -101,7 +101,7 @@ class FileEventPattern(BasePattern): raise ValueError(f"Invalid event mask '{mask}'. Valid are: " f"{FILE_EVENTS}") - def _is_valid_sweep(self, sweep: dict[str,Union[int,float,complex]]) -> None: + def _is_valid_sweep(self, sweep: Dict[str,Union[int,float,complex]]) -> None: """Validation check for 'sweep' variable from main constructor.""" return super()._is_valid_sweep(sweep) @@ -124,8 +124,8 @@ class WatchdogMonitor(BaseMonitor): #A lock to solve race conditions on '_rules' _rules_lock:threading.Lock - def __init__(self, base_dir:str, patterns:dict[str,FileEventPattern], - recipes:dict[str,BaseRecipe], autostart=False, settletime:int=1, + def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern], + recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1, print:Any=sys.stdout, logging:int=0)->None: """WatchdogEventHandler Constructor. This uses the watchdog module to monitor a directory and all its sub-directories. Watchdog will provide @@ -189,7 +189,8 @@ class WatchdogMonitor(BaseMonitor): # Use regex to match event paths against rule paths target_path = rule.pattern.triggering_path recursive_regexp = translate(target_path) - direct_regexp = recursive_regexp.replace('.*', '[^/]*') + direct_regexp = recursive_regexp.replace( + '.*', '[^'+ os.path.sep +']*') recursive_hit = match(recursive_regexp, handle_path) direct_hit = match(direct_regexp, handle_path) @@ -261,7 +262,7 @@ class WatchdogMonitor(BaseMonitor): else: self._identify_lost_rules(lost_pattern=pattern) - def get_patterns(self)->dict[str,FileEventPattern]: + def get_patterns(self)->Dict[str,FileEventPattern]: """Function to get a dict of the currently defined patterns of the monitor. Note that the result is deep-copied, and so can be manipulated without directly manipulating the internals of the monitor.""" @@ -324,7 +325,7 @@ class WatchdogMonitor(BaseMonitor): else: self._identify_lost_rules(lost_recipe=recipe) - def get_recipes(self)->dict[str,BaseRecipe]: + def get_recipes(self)->Dict[str,BaseRecipe]: """Function to get a dict of the currently defined recipes of the monitor. Note that the result is deep-copied, and so can be manipulated without directly manipulating the internals of the monitor.""" @@ -338,7 +339,7 @@ class WatchdogMonitor(BaseMonitor): self._recipes_lock.release() return to_return - def get_rules(self)->dict[str,BaseRule]: + def get_rules(self)->Dict[str,BaseRule]: """Function to get a dict of the currently defined rules of the monitor. Note that the result is deep-copied, and so can be manipulated without directly manipulating the internals of the monitor.""" @@ -450,12 +451,12 @@ class WatchdogMonitor(BaseMonitor): automatically called during initialisation.""" valid_existing_dir_path(base_dir) - def _is_valid_patterns(self, patterns:dict[str,FileEventPattern])->None: + def _is_valid_patterns(self, patterns:Dict[str,FileEventPattern])->None: """Validation check for 'patterns' variable from main constructor. Is automatically called during initialisation.""" valid_dict(patterns, str, FileEventPattern, min_length=0, strict=False) - def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None: + def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None: """Validation check for 'recipes' variable from main constructor. Is automatically called during initialisation.""" valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False) @@ -511,8 +512,8 @@ class WatchdogEventHandler(PatternMatchingEventHandler): # A time to wait per event path, during which extra events are discared _settletime:int # TODO clean this struct occasionally - # A dict of recent job timestamps - _recent_jobs:dict[str, Any] + # A Dict of recent job timestamps + _recent_jobs:Dict[str, Any] # A lock to solve race conditions on '_recent_jobs' _recent_jobs_lock:threading.Lock def __init__(self, monitor:WatchdogMonitor, settletime:int=1): diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index db82ff3..2550252 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -9,7 +9,7 @@ import os import nbformat import sys -from typing import Any, Tuple +from typing import Any, Tuple, Dict from core.correctness.validation import check_type, valid_string, \ valid_dict, valid_path, valid_existing_dir_path, setup_debugging, \ @@ -28,8 +28,8 @@ from core.meow import BaseRecipe, BaseHandler class JupyterNotebookRecipe(BaseRecipe): # A path to the jupyter notebook used to create this recipe source:str - def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={}, - requirements:dict[str,Any]={}, source:str=""): + def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, + requirements:Dict[str,Any]={}, source:str=""): """JupyterNotebookRecipe Constructor. This is used to execute analysis code using the papermill module.""" super().__init__(name, recipe, parameters, requirements) @@ -41,20 +41,20 @@ class JupyterNotebookRecipe(BaseRecipe): if source: valid_path(source, extension=".ipynb", min_length=0) - def _is_valid_recipe(self, recipe:dict[str,Any])->None: + def _is_valid_recipe(self, recipe:Dict[str,Any])->None: """Validation check for 'recipe' variable from main constructor. Called within parent BaseRecipe constructor.""" - check_type(recipe, dict) + check_type(recipe, Dict) nbformat.validate(recipe) - def _is_valid_parameters(self, parameters:dict[str,Any])->None: + def _is_valid_parameters(self, parameters:Dict[str,Any])->None: """Validation check for 'parameters' variable from main constructor. Called within parent BaseRecipe constructor.""" valid_dict(parameters, str, Any, strict=False, min_length=0) for k in parameters.keys(): valid_string(k, VALID_VARIABLE_NAME_CHARS) - def _is_valid_requirements(self, requirements:dict[str,Any])->None: + def _is_valid_requirements(self, requirements:Dict[str,Any])->None: """Validation check for 'requirements' variable from main constructor. Called within parent BaseRecipe constructor.""" valid_dict(requirements, str, Any, strict=False, min_length=0) @@ -86,7 +86,7 @@ class PapermillHandler(BaseHandler): print_debug(self._print_target, self.debug_level, "Created new PapermillHandler instance", DEBUG_INFO) - def handle(self, event:dict[str,Any])->None: + def handle(self, event:Dict[str,Any])->None: """Function called to handle a given event.""" print_debug(self._print_target, self.debug_level, f"Handling event {event[EVENT_PATH]}", DEBUG_INFO) @@ -112,7 +112,7 @@ class PapermillHandler(BaseHandler): yaml_dict[value[0]] = value[1] self.setup_job(event, yaml_dict) - def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]: + def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an event defintion, if this handler can process it or not. This handler accepts events from watchdog with jupyter notebook recipes.""" @@ -135,7 +135,7 @@ class PapermillHandler(BaseHandler): constructor.""" valid_existing_dir_path(output_dir, allow_base=True) - def setup_job(self, event:dict[str,Any], yaml_dict:dict[str,Any])->None: + def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None: """Function to set up new job dict and send it to the runner to be executed.""" meow_job = create_job( diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index ebccccd..dc95a0b 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -8,7 +8,7 @@ Author(s): David Marchant import os import sys -from typing import Any, Tuple +from typing import Any, Tuple, Dict, List from core.correctness.validation import check_script, valid_string, \ valid_dict, valid_event, valid_existing_dir_path, setup_debugging @@ -24,31 +24,32 @@ from core.meow import BaseRecipe, BaseHandler class PythonRecipe(BaseRecipe): - def __init__(self, name:str, recipe:list[str], parameters:dict[str,Any]={}, - requirements:dict[str,Any]={}): + def __init__(self, name:str, recipe:List[str], parameters:Dict[str,Any]={}, + requirements:Dict[str,Any]={}): """PythonRecipe Constructor. This is used to execute python analysis code.""" super().__init__(name, recipe, parameters, requirements) - def _is_valid_recipe(self, recipe:list[str])->None: + def _is_valid_recipe(self, recipe:List[str])->None: """Validation check for 'recipe' variable from main constructor. Called within parent BaseRecipe constructor.""" check_script(recipe) - def _is_valid_parameters(self, parameters:dict[str,Any])->None: + def _is_valid_parameters(self, parameters:Dict[str,Any])->None: """Validation check for 'parameters' variable from main constructor. Called within parent BaseRecipe constructor.""" valid_dict(parameters, str, Any, strict=False, min_length=0) for k in parameters.keys(): valid_string(k, VALID_VARIABLE_NAME_CHARS) - def _is_valid_requirements(self, requirements:dict[str,Any])->None: + def _is_valid_requirements(self, requirements:Dict[str,Any])->None: """Validation check for 'requirements' variable from main constructor. Called within parent BaseRecipe constructor.""" valid_dict(requirements, str, Any, strict=False, min_length=0) for k in requirements.keys(): valid_string(k, VALID_VARIABLE_NAME_CHARS) + class PythonHandler(BaseHandler): # TODO move me to base handler # handler directory to setup jobs in @@ -75,7 +76,7 @@ class PythonHandler(BaseHandler): print_debug(self._print_target, self.debug_level, "Created new PythonHandler instance", DEBUG_INFO) - def handle(self, event:dict[str,Any])->None: + def handle(self, event:Dict[str,Any])->None: """Function called to handle a given event.""" print_debug(self._print_target, self.debug_level, f"Handling event {event[EVENT_PATH]}", DEBUG_INFO) @@ -101,7 +102,7 @@ class PythonHandler(BaseHandler): yaml_dict[value[0]] = value[1] self.setup_job(event, yaml_dict) - def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]: + def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an event defintion, if this handler can process it or not. This handler accepts events from watchdog with Python recipes""" @@ -124,7 +125,7 @@ class PythonHandler(BaseHandler): constructor.""" valid_existing_dir_path(output_dir, allow_base=True) - def setup_job(self, event:dict[str,Any], yaml_dict:dict[str,Any])->None: + def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None: """Function to set up new job dict and send it to the runner to be executed.""" meow_job = create_job( @@ -175,6 +176,7 @@ class PythonHandler(BaseHandler): # Send job directory, as actual definitons will be read from within it self.to_runner.send(job_dir) + # Papermill job execution code, to be run within the conductor def python_job_func(job): # Requires own imports as will be run in its own execution environment diff --git a/tests/shared.py b/tests/shared.py index d0caf50..004e4d1 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -3,6 +3,8 @@ This file contains shared functions and variables used within multiple tests. Author(s): David Marchant """ +import os + from core.functionality import make_dir, rmtree @@ -30,8 +32,8 @@ COMPLETE_PYTHON_SCRIPT = [ "import os", "# Setup parameters", "num = 1000", - "infile = 'somehere/particular'", - "outfile = 'nowhere/particular'", + "infile = 'somehere"+ os.path.sep +"particular'", + "outfile = 'nowhere"+ os.path.sep +"particular'", "", "with open(infile, 'r') as file:", " s = float(file.read())", @@ -131,9 +133,9 @@ APPENDING_NOTEBOOK = { "# The line to append\n", "extra = 'This line comes from a default pattern'\n", "# Data input file location\n", - "infile = 'start/alpha.txt'\n", + "infile = 'start"+ os.path.sep +"alpha.txt'\n", "# Output file location\n", - "outfile = 'first/alpha.txt'" + "outfile = 'first"+ os.path.sep +"alpha.txt'" ] }, { @@ -216,9 +218,9 @@ ADDING_NOTEBOOK = { "# Amount to add to data\n", "extra = 10\n", "# Data input file location\n", - "infile = 'example_data/data_0.npy'\n", + "infile = 'example_data"+ os.path.sep +"data_0.npy'\n", "# Output file location\n", - "outfile = 'standard_output/data_0.npy'" + "outfile = 'standard_output"+ os.path.sep +"data_0.npy'" ] }, { diff --git a/tests/test_conductors.py b/tests/test_conductors.py index d95ae30..03c9bc3 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -2,6 +2,8 @@ import os import unittest +from typing import Dict + from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \ JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \ META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \ @@ -108,7 +110,7 @@ class MeowTests(unittest.TestCase): meta_path = os.path.join(output_dir, META_FILE) self.assertTrue(os.path.exists(meta_path)) status = read_yaml(meta_path) - self.assertIsInstance(status, dict) + self.assertIsInstance(status, Dict) self.assertIn(JOB_STATUS, status) self.assertEqual(status[JOB_STATUS], STATUS_DONE) @@ -196,7 +198,7 @@ class MeowTests(unittest.TestCase): meta_path = os.path.join(output_dir, META_FILE) self.assertTrue(os.path.exists(meta_path)) status = read_yaml(meta_path) - self.assertIsInstance(status, dict) + self.assertIsInstance(status, Dict) self.assertIn(JOB_STATUS, status) self.assertEqual(status[JOB_STATUS], STATUS_DONE) diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 26b5500..58da48e 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -383,28 +383,38 @@ class CorrectnessTests(unittest.TestCase): } replaced = replace_keywords( - test_dict, "job_id", "base/src/dir/file.ext", "base/monitor/dir") + test_dict, + "job_id", + os.path.join("base", "src", "dir", "file.ext"), + os.path.join("base", "monitor", "dir") + ) self.assertIsInstance(replaced, dict) self.assertEqual(len(test_dict.keys()), len(replaced.keys())) for k in test_dict.keys(): self.assertIn(k, replaced) - self.assertEqual(replaced["A"], "--base/src/dir/file.ext--") - self.assertEqual(replaced["B"], "--../../src/dir/file.ext--") - self.assertEqual(replaced["C"], "--base/src/dir--") - self.assertEqual(replaced["D"], "--../../src/dir--") + self.assertEqual(replaced["A"], + os.path.join("--base", "src", "dir", "file.ext--")) + self.assertEqual(replaced["B"], + os.path.join("--..", "..", "src", "dir", "file.ext--")) + self.assertEqual(replaced["C"], + os.path.join("--base", "src", "dir--")) + self.assertEqual(replaced["D"], + os.path.join("--..", "..", "src", "dir--")) self.assertEqual(replaced["E"], "--file.ext--") self.assertEqual(replaced["F"], "--file--") - self.assertEqual(replaced["G"], "--base/monitor/dir--") + self.assertEqual(replaced["G"], + os.path.join("--base", "monitor", "dir--")) self.assertEqual(replaced["H"], "--.ext--") self.assertEqual(replaced["I"], "--job_id--") self.assertEqual(replaced["J"], - "--base/src/dir/file.ext-base/src/dir/file.ext--") - self.assertEqual(replaced["K"], "base/src/dir/file.ext") + os.path.join("--base", "src", "dir", "file.ext-base", "src", "dir", "file.ext--")) + self.assertEqual(replaced["K"], + os.path.join("base", "src", "dir", "file.ext")) self.assertEqual(replaced["L"], - "--base/src/dir/file.ext-../../src/dir/file.ext-base/src/dir-" - "../../src/dir-file.ext-file-base/monitor/dir-.ext-job_id--") + os.path.join("--base", "src", "dir", "file.ext-..", "..", "src", "dir", "file.ext-base", "src", "dir-" + "..", "..", "src", "dir-file.ext-file-base", "monitor", "dir-.ext-job_id--")) self.assertEqual(replaced["M"], "A") self.assertEqual(replaced["N"], 1) @@ -498,30 +508,31 @@ data""" '"metadata": {}, "outputs": [], "source": ["# Default parameters ' 'values\\n", "# The line to append\\n", "extra = \'This line ' 'comes from a default pattern\'\\n", "# Data input file ' - 'location\\n", "infile = \'start/alpha.txt\'\\n", "# Output file ' - 'location\\n", "outfile = \'first/alpha.txt\'"]}, {"cell_type": ' + 'location\\n", "infile = \'start'+ os.path.sep +'alpha.txt\'\\n", ' + '"# Output file location\\n", "outfile = \'first'+ os.path.sep + +'alpha.txt\'"]}, {"cell_type": "code", "execution_count": null, ' + '"metadata": {}, "outputs": [], "source": ["# load in dataset. ' + 'This should be a text file\\n", "with open(infile) as ' + 'input_file:\\n", " data = input_file.read()"]}, {"cell_type": ' '"code", "execution_count": null, "metadata": {}, "outputs": [], ' - '"source": ["# load in dataset. This should be a text file\\n", ' - '"with open(infile) as input_file:\\n", " data = ' - 'input_file.read()"]}, {"cell_type": "code", "execution_count": ' - 'null, "metadata": {}, "outputs": [], "source": ["# Append the ' - 'line\\n", "appended = data + \'\\\\n\' + extra"]}, {"cell_type": ' - '"code", "execution_count": null, "metadata": {}, "outputs": [], ' - '"source": ["import os\\n", "\\n", "# Create output directory if ' - 'it doesn\'t exist\\n", "output_dir_path = ' - 'os.path.dirname(outfile)\\n", "\\n", "if output_dir_path:\\n", ' - '" os.makedirs(output_dir_path, exist_ok=True)\\n", "\\n", "# ' - 'Save added array as new dataset\\n", "with open(outfile, \'w\') ' - 'as output_file:\\n", " output_file.write(appended)"]}], ' - '"metadata": {"kernelspec": {"display_name": "Python 3", ' - '"language": "python", "name": "python3"}, "language_info": ' - '{"codemirror_mode": {"name": "ipython", "version": 3}, ' - '"file_extension": ".py", "mimetype": "text/x-python", "name": ' - '"python", "nbconvert_exporter": "python", "pygments_lexer": ' - '"ipython3", "version": "3.10.6 (main, Nov 14 2022, 16:10:14) ' - '[GCC 11.3.0]"}, "vscode": {"interpreter": {"hash": ' - '"916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1' - '"}}}, "nbformat": 4, "nbformat_minor": 4}' + '"source": ["# Append the line\\n", "appended = data + \'\\\\n\' ' + '+ extra"]}, {"cell_type": "code", "execution_count": null, ' + '"metadata": {}, "outputs": [], "source": ["import os\\n", "\\n", ' + '"# Create output directory if it doesn\'t exist\\n", ' + '"output_dir_path = os.path.dirname(outfile)\\n", "\\n", ' + '"if output_dir_path:\\n", " os.makedirs(output_dir_path, ' + 'exist_ok=True)\\n", "\\n", "# Save added array as new ' + 'dataset\\n", "with open(outfile, \'w\') as output_file:\\n", " ' + 'output_file.write(appended)"]}], "metadata": {"kernelspec": ' + '{"display_name": "Python 3", "language": "python", "name": ' + '"python3"}, "language_info": {"codemirror_mode": {"name": ' + '"ipython", "version": 3}, "file_extension": ".py", "mimetype": ' + '"text/x-python", "name": "python", "nbconvert_exporter": ' + '"python", "pygments_lexer": "ipython3", "version": "3.10.6 ' + '(main, Nov 14 2022, 16:10:14) [GCC 11.3.0]"}, "vscode": ' + '{"interpreter": {"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c' + '65e1c03b4aa3499c5328201f1"}}}, "nbformat": 4, "nbformat_minor": ' + '4}' ] self.assertEqual(data, expected_bytes) diff --git a/tests/test_meow.py b/tests/test_meow.py index 8c3bc60..ca09ab3 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -1,7 +1,7 @@ import unittest -from typing import Any, Union, Tuple +from typing import Any, Union, Tuple, Dict from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \ @@ -67,7 +67,7 @@ class MeowTests(unittest.TestCase): def _is_valid_output(self, outputs:Any)->None: pass def _is_valid_sweep(self, - sweep:dict[str,Union[int,float,complex]])->None: + sweep:Dict[str,Union[int,float,complex]])->None: pass FullPattern("name", "", "", "", "") @@ -188,7 +188,7 @@ class MeowTests(unittest.TestCase): valid_recipe_two.name: valid_recipe_two } rules = create_rules(patterns, recipes) - self.assertIsInstance(rules, dict) + self.assertIsInstance(rules, Dict) self.assertEqual(len(rules), 2) for k, rule in rules.items(): self.assertIsInstance(k, str) @@ -229,9 +229,9 @@ class MeowTests(unittest.TestCase): pass def stop(self): pass - def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None: + def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None: pass - def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None: + def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None: pass def add_pattern(self, pattern:BasePattern)->None: pass @@ -274,7 +274,7 @@ class MeowTests(unittest.TestCase): pass def _is_valid_inputs(self, inputs:Any)->None: pass - def valid_handle_criteria(self, event:dict[str,Any] + def valid_handle_criteria(self, event:Dict[str,Any] )->Tuple[bool,str]: pass @@ -292,10 +292,10 @@ class MeowTests(unittest.TestCase): TestConductor() class FullTestConductor(BaseConductor): - def execute(self, job:dict[str,Any])->None: + def execute(self, job:Dict[str,Any])->None: pass - def valid_execute_criteria(self, job:dict[str,Any] + def valid_execute_criteria(self, job:Dict[str,Any] )->Tuple[bool,str]: pass diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 2456edd..ed59ccb 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -245,7 +245,10 @@ class CorrectnessTests(unittest.TestCase): # Test WatchdogMonitor identifies expected events in sub directories def testMonitoring(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={}) recipe = JupyterNotebookRecipe( "recipe_one", BAREBONES_NOTEBOOK) @@ -304,7 +307,10 @@ class CorrectnessTests(unittest.TestCase): # Test WatchdogMonitor identifies fake events for retroactive patterns def testMonitoringRetroActive(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={}) recipe = JupyterNotebookRecipe( "recipe_one", BAREBONES_NOTEBOOK) @@ -368,10 +374,16 @@ class CorrectnessTests(unittest.TestCase): # Test WatchdogMonitor get_patterns function def testMonitorGetPatterns(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={}) pattern_two = FileEventPattern( - "pattern_two", "start/B.txt", "recipe_two", "infile", + "pattern_two", + os.path.join("start", "B.txt"), + "recipe_two", + "infile", parameters={}) wm = WatchdogMonitor( @@ -395,10 +407,16 @@ class CorrectnessTests(unittest.TestCase): # Test WatchdogMonitor add_pattern function def testMonitorAddPattern(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={}) pattern_two = FileEventPattern( - "pattern_two", "start/B.txt", "recipe_two", "infile", + "pattern_two", + os.path.join("start", "B.txt"), + "recipe_two", + "infile", parameters={}) wm = WatchdogMonitor( @@ -440,10 +458,16 @@ class CorrectnessTests(unittest.TestCase): # Test WatchdogMonitor update_patterns function def testMonitorUpdatePattern(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={}) pattern_two = FileEventPattern( - "pattern_two", "start/B.txt", "recipe_two", "infile", + "pattern_two", + os.path.join("start", "B.txt"), + "recipe_two", + "infile", parameters={}) wm = WatchdogMonitor( @@ -502,10 +526,16 @@ class CorrectnessTests(unittest.TestCase): # Test WatchdogMonitor remove_patterns function def testMonitorRemovePattern(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={}) pattern_two = FileEventPattern( - "pattern_two", "start/B.txt", "recipe_two", "infile", + "pattern_two", + os.path.join("start", "B.txt"), + "recipe_two", + "infile", parameters={}) wm = WatchdogMonitor( @@ -707,10 +737,16 @@ class CorrectnessTests(unittest.TestCase): # Test WatchdogMonitor get_rules function def testMonitorGetRules(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={}) pattern_two = FileEventPattern( - "pattern_two", "start/B.txt", "recipe_two", "infile", + "pattern_two", + os.path.join("start", "B.txt"), + "recipe_two", + "infile", parameters={}) recipe_one = JupyterNotebookRecipe( "recipe_one", BAREBONES_NOTEBOOK) diff --git a/tests/test_recipes.py b/tests/test_recipes.py index eac2662..3f8c699 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -4,6 +4,7 @@ import os import unittest from multiprocessing import Pipe +from typing import Dict from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \ @@ -391,7 +392,7 @@ class JupyterNotebookTests(unittest.TestCase): meta_path = os.path.join(job_dir, META_FILE) self.assertTrue(os.path.exists(meta_path)) status = read_yaml(meta_path) - self.assertIsInstance(status, dict) + self.assertIsInstance(status, Dict) self.assertIn(JOB_STATUS, status) self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS]) @@ -759,7 +760,7 @@ class PythonTests(unittest.TestCase): self.assertTrue(os.path.exists(meta_path)) status = read_yaml(meta_path) - self.assertIsInstance(status, dict) + self.assertIsInstance(status, Dict) self.assertIn(JOB_STATUS, status) self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS]) self.assertNotIn(JOB_ERROR, status) diff --git a/tests/test_runner.py b/tests/test_runner.py index e5fd26a..8c9a79a 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -121,10 +121,13 @@ class MeowTests(unittest.TestCase): # Test single meow papermill job execution def testMeowRunnerPapermillExecution(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={ "extra":"A line from a test Pattern", - "outfile":"{VGRID}/output/{FILENAME}" + "outfile":os.path.join("{VGRID}", "output", "{FILENAME}") }) recipe = JupyterNotebookRecipe( "recipe_one", APPENDING_NOTEBOOK) @@ -205,16 +208,19 @@ class MeowTests(unittest.TestCase): # Test meow papermill job chaining within runner def testMeowRunnerLinkedPapermillExecution(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", + os.path.join("start", "A.txt"), + "recipe_one", + "infile", parameters={ "extra":"A line from Pattern 1", - "outfile":"{VGRID}/middle/{FILENAME}" + "outfile":os.path.join("{VGRID}", "middle", "{FILENAME}") }) pattern_two = FileEventPattern( - "pattern_two", "middle/A.txt", "recipe_one", "infile", + "pattern_two", os.path.join("middle", "A.txt"), "recipe_one", "infile", parameters={ "extra":"A line from Pattern 2", - "outfile":"{VGRID}/output/{FILENAME}" + "outfile":os.path.join("{VGRID}", "output", "{FILENAME}") }) recipe = JupyterNotebookRecipe( "recipe_one", APPENDING_NOTEBOOK) @@ -314,10 +320,10 @@ class MeowTests(unittest.TestCase): # Test single meow python job execution def testMeowRunnerPythonExecution(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", os.path.join("start", "A.txt"), "recipe_one", "infile", parameters={ "num":10000, - "outfile":"{VGRID}/output/{FILENAME}" + "outfile":os.path.join("{VGRID}", "output", "{FILENAME}") }) recipe = PythonRecipe( "recipe_one", COMPLETE_PYTHON_SCRIPT @@ -403,16 +409,16 @@ class MeowTests(unittest.TestCase): # Test meow python job chaining within runner def testMeowRunnerLinkedPythonExecution(self)->None: pattern_one = FileEventPattern( - "pattern_one", "start/A.txt", "recipe_one", "infile", + "pattern_one", os.path.join("start", "A.txt"), "recipe_one", "infile", parameters={ "num":250, - "outfile":"{VGRID}/middle/{FILENAME}" + "outfile":os.path.join("{VGRID}", "middle", "{FILENAME}") }) pattern_two = FileEventPattern( - "pattern_two", "middle/A.txt", "recipe_one", "infile", + "pattern_two", os.path.join("middle", "A.txt"), "recipe_one", "infile", parameters={ "num":40, - "outfile":"{VGRID}/output/{FILENAME}" + "outfile":os.path.join("{VGRID}", "output", "{FILENAME}") }) recipe = PythonRecipe( "recipe_one", COMPLETE_PYTHON_SCRIPT diff --git a/tests/test_validation.py b/tests/test_validation.py index 6829f2b..8c2811e 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -239,9 +239,10 @@ class CorrectnessTests(unittest.TestCase): with self.assertRaises(ValueError): valid_non_existing_path("first") - make_dir("first/second") + test_path = os.path.join("first", "second") + make_dir(test_path) with self.assertRaises(ValueError): - valid_non_existing_path("first/second") + valid_non_existing_path(test_path) # TODO modify so tests for actual rule values # Test valid_event can check given event dictionary