NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated code so should work on windows, with exception of waiting on multiple connections

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-08 14:46:03 +01:00
parent d787e37adc
commit a1451881ae
17 changed files with 280 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
core/correctness/validation.py
View File

@ -8,7 +8,8 @@ Author(s): David Marchant
from datetime import datetime
from inspect import signature
from os.path import sep, exists, isfile, isdir, dirname
from typing import Any, _SpecialForm, Union, Tuple, get_origin, get_args
from typing import Any, _SpecialForm, Union, Tuple, Type, Dict, List, \
get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
@ -33,7 +34,7 @@ WATCHDOG_EVENT_KEYS = {
# Required keys in job dict
JOB_KEYS = {
JOB_TYPE: str,
JOB_EVENT: dict,
JOB_EVENT: Dict,
JOB_ID: str,
JOB_PATTERN: Any,
JOB_RECIPE: Any,
@ -42,7 +43,7 @@ JOB_KEYS = {
JOB_CREATE_TIME: datetime,
}
def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
or_none:bool=False)->None:
"""Checks if a given variable is of the expected type. Raises TypeError or
ValueError as appropriate if any issues are encountered."""
@ -52,6 +53,11 @@ def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
if get_origin(expected_type) is Union:
type_list = list(get_args(expected_type))
type_list = type_list + alt_types
# # If we have any types from typing, then update to allow checks against
# # their base types too
# for t in type_list:
# if get_origin(t):
# type_list.append(get_origin(t))
# Only accept None if explicitly allowed
if variable is None:
@ -130,16 +136,16 @@ def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
"%s" % (char, valid_chars)
)
def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type,
required_keys:list[Any]=[], optional_keys:list[Any]=[],
def valid_dict(variable:Dict[Any, Any], key_type:Type, value_type:Type,
required_keys:List[Any]=[], optional_keys:List[Any]=[],
strict:bool=True, min_length:int=1)->None:
"""Checks that a given dictionary is valid. Key and Value types are
enforced, as are required and optional keys. Will raise ValueError,
TypeError or KeyError depending on the problem encountered."""
# Validate inputs
check_type(variable, dict)
check_type(key_type, type, alt_types=[_SpecialForm])
check_type(value_type, type, alt_types=[_SpecialForm])
check_type(variable, Dict)
check_type(key_type, Type, alt_types=[_SpecialForm])
check_type(value_type, Type, alt_types=[_SpecialForm])
check_type(required_keys, list)
check_type(optional_keys, list)
check_type(strict, bool)
@ -172,11 +178,11 @@ def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type,
raise ValueError(f"Unexpected key '{k}' should not be present "
f"in dict '{variable}'")
def valid_list(variable:list[Any], entry_type:type,
alt_types:list[type]=[], min_length:int=1)->None:
def valid_list(variable:List[Any], entry_type:Type,
alt_types:List[Type]=[], min_length:int=1)->None:
"""Checks that a given list is valid. Value types are checked and a
ValueError or TypeError is raised if a problem is encountered."""
check_type(variable, list)
check_type(variable, List)
# Check length meets minimum
if len(variable) < min_length:
@ -256,11 +262,11 @@ def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:
return print, logging
def valid_meow_dict(meow_dict:dict[str,Any], msg:str,
keys:dict[str,type])->None:
def valid_meow_dict(meow_dict:Dict[str,Any], msg:str,
keys:Dict[str,Type])->None:
"""Check given dictionary expresses a meow construct. This won't do much
directly, but is called by more specific validation functions."""
check_type(meow_dict, dict)
check_type(meow_dict, Dict)
# Check we have all the required keys, and they are all of the expected
# type
for key, value_type in keys.items():
@ -268,13 +274,13 @@ def valid_meow_dict(meow_dict:dict[str,Any], msg:str,
raise KeyError(f"{msg} require key '{key}'")
check_type(meow_dict[key], value_type)
def valid_event(event:dict[str,Any])->None:
def valid_event(event:Dict[str,Any])->None:
"""Check that a given dict expresses a meow event."""
valid_meow_dict(event, "Event", EVENT_KEYS)
def valid_job(job:dict[str,Any])->None:
def valid_job(job:Dict[str,Any])->None:
"""Check that a given dict expresses a meow job."""
valid_meow_dict(job, "Job", JOB_KEYS)
def valid_watchdog_event(event:dict[str,Any])->None:
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

44
core/functionality.py
View File

@ -7,11 +7,12 @@ import os
import yaml
from datetime import datetime
from typing import List
from multiprocessing.connection import Connection, wait as multi_wait
from multiprocessing.queues import Queue
from papermill.translators import papermill_translators
from typing import Any
from typing import Any, Dict
from random import SystemRandom
from core.correctness.validation import check_type, valid_existing_file_path, \
@ -36,7 +37,7 @@ KEYWORD_JOB = "{JOB}"
#TODO Make this guaranteed unique
def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
def generate_id(prefix:str="", length:int=16, existing_ids:List[str]=[],
charset:str=CHAR_UPPERCASE+CHAR_LOWERCASE, attempts:int=24):
random_length = max(length - len(prefix), 0)
for _ in range(attempts):
@ -47,10 +48,12 @@ def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
raise ValueError(f"Could not generate ID unique from '{existing_ids}' "
f"using values '{charset}' and length of '{length}'.")
def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]:
def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:
all_connections = [i for i in inputs if type(i) is Connection] \
+ [i._reader for i in inputs if type(i) is Queue]
for i in inputs:
print(type(i))
ready = multi_wait(all_connections)
ready_inputs = [i for i in inputs if \
(type(i) is Connection and i in ready) \
@ -72,7 +75,6 @@ def _get_file_sha256(file_path):
def get_file_hash(file_path:str, hash:str):
check_type(hash, str)
import os
valid_existing_file_path(file_path)
valid_hashes = {
@ -80,7 +82,7 @@ def get_file_hash(file_path:str, hash:str):
}
if hash not in valid_hashes:
raise KeyError(f"Cannot use hash '{hash}'. Valid are "
"'{list(valid_hashes.keys())}")
f"'{list(valid_hashes.keys())}")
return valid_hashes[hash](file_path)
@ -169,7 +171,7 @@ def read_notebook(filepath:str):
with open(filepath, 'r') as read_file:
return json.load(read_file)
def write_notebook(source:dict[str,Any], filename:str):
def write_notebook(source:Dict[str,Any], filename:str):
"""
Writes the given notebook source code to a given filename.
@ -183,10 +185,10 @@ def write_notebook(source:dict[str,Any], filename:str):
json.dump(source, job_file)
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any],
parameters:dict[str,Any], expand_env_values:bool=False)->dict[str,Any]:
def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any],
parameters:Dict[str,Any], expand_env_values:bool=False)->Dict[str,Any]:
nbformat.validate(jupyter_notebook)
check_type(parameters, dict)
check_type(parameters, Dict)
if jupyter_notebook["nbformat"] != 4:
raise Warning(
@ -256,10 +258,10 @@ def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any],
return output_notebook
def parameterize_python_script(script:list[str], parameters:dict[str,Any],
expand_env_values:bool=False)->dict[str,Any]:
def parameterize_python_script(script:List[str], parameters:Dict[str,Any],
expand_env_values:bool=False)->Dict[str,Any]:
check_script(script)
check_type(parameters, dict)
check_type(parameters, Dict)
output_script = copy.deepcopy(script)
@ -302,8 +304,8 @@ def print_debug(print_target, debug_level, msg, level)->None:
status = "WARNING"
print(f"{status}: {msg}", file=print_target)
def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
monitor_base:str)->dict[str,str]:
def replace_keywords(old_dict:Dict[str,str], job_id:str, src_path:str,
monitor_base:str)->Dict[str,str]:
"""Function to replace all MEOW magic words in a dictionary with dynamic
values."""
new_dict = {}
@ -332,8 +334,8 @@ def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
return new_dict
def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={}
)->dict[Any,Any]:
def create_event(event_type:str, path:str, rule:Any, extras:Dict[Any,Any]={}
)->Dict[Any,Any]:
"""Function to create a MEOW dictionary."""
return {
**extras,
@ -343,7 +345,7 @@ def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={}
}
def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
extras:dict[Any,Any]={})->dict[Any,Any]:
extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW event dictionary."""
return create_event(
EVENT_TYPE_WATCHDOG,
@ -358,8 +360,8 @@ def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
}
)
def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
)->dict[Any,Any]:
def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
)->Dict[Any,Any]:
"""Function to create a MEOW job dictionary."""
job_dict = {
#TODO compress event?
@ -376,7 +378,7 @@ def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
return {**extras, **job_dict}
def lines_to_string(lines:list[str])->str:
def lines_to_string(lines:List[str])->str:
"""Function to convert a list of str lines, into one continuous string
separated by newline characters"""
return "\n".join(lines)

66
core/meow.py
View File

@ -12,7 +12,7 @@ import itertools
import sys
from copy import deepcopy
from typing import Any, Union, Tuple
from typing import Any, Union, Tuple, Dict, List
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
@ -28,11 +28,11 @@ class BaseRecipe:
# Actual code to run
recipe:Any
# Possible parameters that could be overridden by a Pattern
parameters:dict[str, Any]
parameters:Dict[str, Any]
# Additional configuration options
requirements:dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={},
requirements:dict[str,Any]={}):
requirements:Dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={},
requirements:Dict[str,Any]={}):
"""BaseRecipe Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
@ -84,14 +84,14 @@ class BasePattern:
# An identifier of a recipe
recipe:str
# Parameters to be overridden in the recipe
parameters:dict[str,Any]
parameters:Dict[str,Any]
# Parameters showing the potential outputs of a recipe
outputs:dict[str,Any]
outputs:Dict[str,Any]
# A collection of variables to be swept over for job scheduling
sweep:dict[str,Any]
sweep:Dict[str,Any]
def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={},
outputs:dict[str,Any]={}, sweep:dict[str,Any]={}):
def __init__(self, name:str, recipe:str, parameters:Dict[str,Any]={},
outputs:Dict[str,Any]={}, sweep:Dict[str,Any]={}):
"""BasePattern Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
@ -138,11 +138,11 @@ class BasePattern:
be implemented by any child class."""
pass
def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None:
def _is_valid_sweep(self, sweep:Dict[str,Union[int,float,complex]])->None:
"""Validation check for 'sweep' variable from main constructor. This
function is implemented to check for the types given in the signature,
and must be overridden if these differ."""
check_type(sweep, dict)
check_type(sweep, Dict)
if not sweep:
return
for _, v in sweep.items():
@ -175,7 +175,7 @@ class BasePattern:
"value where the end point is smaller than the start."
)
def expand_sweeps(self)->list[Tuple[str,Any]]:
def expand_sweeps(self)->List[Tuple[str,Any]]:
"""Function to get all combinations of sweep parameters"""
values_dict = {}
# get a collection of a individual sweep values
@ -259,17 +259,17 @@ class BaseRule:
class BaseMonitor:
# A collection of patterns
_patterns: dict[str, BasePattern]
_patterns: Dict[str, BasePattern]
# A collection of recipes
_recipes: dict[str, BaseRecipe]
_recipes: Dict[str, BaseRecipe]
# A collection of rules derived from _patterns and _recipes
_rules: dict[str, BaseRule]
_rules: Dict[str, BaseRule]
# A channel for sending messages to the runner. Note that this is not
# initialised within the constructor, but within the runner when passed the
# monitor is passed to it.
to_runner: VALID_CHANNELS
def __init__(self, patterns:dict[str,BasePattern],
recipes:dict[str,BaseRecipe])->None:
def __init__(self, patterns:Dict[str,BasePattern],
recipes:Dict[str,BaseRecipe])->None:
"""BaseMonitor Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
@ -302,12 +302,12 @@ class BaseMonitor:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None:
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
"""Validation check for 'patterns' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None:
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
"""Validation check for 'recipes' variable from main constructor. Must
be implemented by any child class."""
pass
@ -337,7 +337,7 @@ class BaseMonitor:
implemented by any child process."""
pass
def get_patterns(self)->dict[str,BasePattern]:
def get_patterns(self)->Dict[str,BasePattern]:
"""Function to get a dictionary of all current pattern definitions.
Must be implemented by any child process."""
pass
@ -357,12 +357,12 @@ class BaseMonitor:
implemented by any child process."""
pass
def get_recipes(self)->dict[str,BaseRecipe]:
def get_recipes(self)->Dict[str,BaseRecipe]:
"""Function to get a dictionary of all current recipe definitions.
Must be implemented by any child process."""
pass
def get_rules(self)->dict[str,BaseRule]:
def get_rules(self)->Dict[str,BaseRule]:
"""Function to get a dictionary of all current rule definitions.
Must be implemented by any child process."""
pass
@ -387,12 +387,12 @@ class BaseHandler:
raise TypeError(msg)
return object.__new__(cls)
def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]:
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. Must be implemented by any child process."""
pass
def handle(self, event:dict[str,Any])->None:
def handle(self, event:Dict[str,Any])->None:
"""Function to handle a given event. Must be implemented by any child
process."""
pass
@ -413,28 +413,28 @@ class BaseConductor:
raise TypeError(msg)
return object.__new__(cls)
def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]:
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
process it or not. Must be implemented by any child process."""
pass
def execute(self, job:dict[str,Any])->None:
def execute(self, job:Dict[str,Any])->None:
"""Function to execute a given job. Must be implemented by any child
process."""
pass
def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
recipes:Union[dict[str,BaseRecipe],list[BaseRecipe]],
new_rules:list[BaseRule]=[])->dict[str,BaseRule]:
def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
"""Function to create any valid rules from a given collection of patterns
and recipes. All inbuilt rule types are considered, with additional
definitions provided through the 'new_rules' variable. Note that any
provided pattern and recipe dictionaries must be keyed with the
corresponding pattern and recipe names."""
# Validation of inputs
check_type(patterns, dict, alt_types=[list])
check_type(recipes, dict, alt_types=[list])
check_type(patterns, Dict, alt_types=[List])
check_type(recipes, Dict, alt_types=[List])
valid_list(new_rules, BaseRule, min_length=0)
# Convert a pattern list to a dictionary
@ -477,7 +477,7 @@ def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
return generated_rules
def create_rule(pattern:BasePattern, recipe:BaseRecipe,
new_rules:list[BaseRule]=[])->BaseRule:
new_rules:List[BaseRule]=[])->BaseRule:
"""Function to create a valid rule from a given pattern and recipe. All
inbuilt rule types are considered, with additional definitions provided
through the 'new_rules' variable."""

34
core/runner.py
View File

@ -12,7 +12,7 @@ import threading
from multiprocessing import Pipe
from random import randrange
from typing import Any, Union
from typing import Any, Union, Dict, List
from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
VALID_CHANNELS, JOB_ID, META_FILE
@ -24,18 +24,18 @@ from core.meow import BaseHandler, BaseMonitor, BaseConductor
class MeowRunner:
# A collection of all monitors in the runner
monitors:list[BaseMonitor]
monitors:List[BaseMonitor]
# A collection of all handlers in the runner
handlers:list[BaseHandler]
handlers:List[BaseHandler]
# A collection of all conductors in the runner
conductors:list[BaseConductor]
conductors:List[BaseConductor]
# A collection of all channels from each monitor
from_monitors: list[VALID_CHANNELS]
from_monitors: List[VALID_CHANNELS]
# A collection of all channels from each handler
from_handlers: list[VALID_CHANNELS]
def __init__(self, monitors:Union[BaseMonitor,list[BaseMonitor]],
handlers:Union[BaseHandler,list[BaseHandler]],
conductors:Union[BaseConductor,list[BaseConductor]],
from_handlers: List[VALID_CHANNELS]
def __init__(self, monitors:Union[BaseMonitor,List[BaseMonitor]],
handlers:Union[BaseHandler,List[BaseHandler]],
conductors:Union[BaseConductor,List[BaseConductor]],
print:Any=sys.stdout, logging:int=0)->None:
"""MeowRunner constructor. This connects all provided monitors,
handlers and conductors according to what events and jobs they produce
@ -178,7 +178,7 @@ class MeowRunner:
]
self.execute_job(conductor, job)
def handle_event(self, handler:BaseHandler, event:dict[str:Any])->None:
def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None:
"""Function for a given handler to handle a given event, without
crashing the runner in the event of a problem."""
print_debug(self._print_target, self.debug_level,
@ -193,7 +193,7 @@ class MeowRunner:
"Something went wrong during handling for event "
f"'{event[EVENT_TYPE]}'. {e}", DEBUG_INFO)
def execute_job(self, conductor:BaseConductor, job:dict[str:Any])->None:
def execute_job(self, conductor:BaseConductor, job:Dict[str,Any])->None:
"""Function for a given conductor to execute a given job, without
crashing the runner in the event of a problem."""
print_debug(self._print_target, self.debug_level,
@ -306,22 +306,22 @@ class MeowRunner:
"Job conductor thread stopped", DEBUG_INFO)
def _is_valid_monitors(self,
monitors:Union[BaseMonitor,list[BaseMonitor]])->None:
monitors:Union[BaseMonitor,List[BaseMonitor]])->None:
"""Validation check for 'monitors' variable from main constructor."""
check_type(monitors, BaseMonitor, alt_types=[list])
check_type(monitors, BaseMonitor, alt_types=[List])
if type(monitors) == list:
valid_list(monitors, BaseMonitor, min_length=1)
def _is_valid_handlers(self,
handlers:Union[BaseHandler,list[BaseHandler]])->None:
handlers:Union[BaseHandler,List[BaseHandler]])->None:
"""Validation check for 'handlers' variable from main constructor."""
check_type(handlers, BaseHandler, alt_types=[list])
check_type(handlers, BaseHandler, alt_types=[List])
if type(handlers) == list:
valid_list(handlers, BaseHandler, min_length=1)
def _is_valid_conductors(self,
conductors:Union[BaseConductor,list[BaseConductor]])->None:
conductors:Union[BaseConductor,List[BaseConductor]])->None:
"""Validation check for 'conductors' variable from main constructor."""
check_type(conductors, BaseConductor, alt_types=[list])
check_type(conductors, BaseConductor, alt_types=[List])
if type(conductors) == list:
valid_list(conductors, BaseConductor, min_length=1)