NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added support for multi-type waiting plus some cleanup

Browse Source
This commit is contained in:
PatchOfScotland
2022-12-15 11:31:51 +01:00
parent 380f7066e1
commit ea9a689b26
12 changed files with 516 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
core/correctness/validation.py
View File

@ -1,10 +1,11 @@
from inspect import signature
from os.path import sep
from typing import Any, _SpecialForm
from typing import Any, _SpecialForm, Union, get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
def check_input(variable:Any, expected_type:type, alt_types:list[type]=[],
def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
or_none:bool=False)->None:
"""
Checks if a given variable is of the expected type. Raises TypeError or
@ -24,23 +25,38 @@ def check_input(variable:Any, expected_type:type, alt_types:list[type]=[],
"""
type_list = [expected_type]
if get_origin(expected_type) is Union:
type_list = list(get_args(expected_type))
type_list = type_list + alt_types
if not or_none:
if expected_type != Any \
and type(variable) not in type_list:
if variable is None:
if or_none == False:
raise TypeError(
'Expected type was %s, got %s'
% (expected_type, type(variable))
)
else:
if expected_type != Any \
and not type(variable) not in type_list \
and not isinstance(variable, type(None)):
raise TypeError(
'Expected type was %s or None, got %s'
% (expected_type, type(variable))
f'Not allowed None for variable. Expected {expected_type}.'
)
else:
return
if expected_type == Any:
return
if not isinstance(variable, tuple(type_list)):
raise TypeError(
'Expected type(s) are %s, got %s'
% (get_args(expected_type), type(variable))
)
def check_implementation(child_func, parent_class):
parent_func = getattr(parent_class, child_func.__name__)
if (child_func == parent_func):
msg = get_not_imp_msg(parent_class, parent_func)
raise NotImplementedError(msg)
child_sig = signature(child_func).parameters
parent_sig = signature(parent_func).parameters
if child_sig.keys() != parent_sig.keys():
msg = get_not_imp_msg(parent_class, parent_func)
raise NotImplementedError(msg)
def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
"""
@ -56,8 +72,8 @@ def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
:return: No return.
"""
check_input(variable, str)
check_input(valid_chars, str)
check_type(variable, str)
check_type(valid_chars, str)
if len(variable) < min_length:
raise ValueError (
@ -74,12 +90,12 @@ def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type,
required_keys:list[Any]=[], optional_keys:list[Any]=[],
strict:bool=True, min_length:int=1)->None:
check_input(variable, dict)
check_input(key_type, type, alt_types=[_SpecialForm])
check_input(value_type, type, alt_types=[_SpecialForm])
check_input(required_keys, list)
check_input(optional_keys, list)
check_input(strict, bool)
check_type(variable, dict)
check_type(key_type, type, alt_types=[_SpecialForm])
check_type(value_type, type, alt_types=[_SpecialForm])
check_type(required_keys, list)
check_type(optional_keys, list)
check_type(strict, bool)
if len(variable) < min_length:
raise ValueError(f"Dictionary '{variable}' is below minimum length of "
@ -106,18 +122,17 @@ def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type,
def valid_list(variable:list[Any], entry_type:type,
alt_types:list[type]=[], min_length:int=1)->None:
check_input(variable, list)
check_type(variable, list)
if len(variable) < min_length:
raise ValueError(f"List '{variable}' is too short. Should be at least "
f"of length {min_length}")
for entry in variable:
check_input(entry, entry_type, alt_types=alt_types)
check_type(entry, entry_type, alt_types=alt_types)
def valid_path(variable:str, allow_base=False, extension:str="", min_length=1):
valid_string(variable, VALID_PATH_CHARS, min_length=min_length)
if not allow_base and variable.startswith(sep):
raise ValueError(f"Cannot accept path '{variable}'. Must be relative.")
if min_length > 0 and extension and not variable.endswith(extension):
if extension and not variable.endswith(extension):
raise ValueError(f"Path '{variable}' does not have required "
f"extension '{extension}'.")

6
core/correctness/vars.py
View File

@ -1,8 +1,12 @@
import os
from multiprocessing import Queue
from multiprocessing.connection import Connection
from inspect import signature
from typing import Union
CHAR_LOWERCASE = 'abcdefghijklmnopqrstuvwxyz'
CHAR_UPPERCASE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
CHAR_NUMERIC = '0123456789'
@ -20,6 +24,8 @@ VALID_JUPYTER_NOTEBOOK_EXTENSIONS = [".ipynb"]
VALID_PATH_CHARS = VALID_NAME_CHARS + "." + os.path.sep
VALID_TRIGGERING_PATH_CHARS = VALID_NAME_CHARS + ".*" + os.path.sep
VALID_CHANNELS = Union[Connection,Queue]
BAREBONES_NOTEBOOK = {
"cells": [],
"metadata": {},

29
core/functionality.py
View File

@ -2,15 +2,15 @@
import sys
import inspect
from multiprocessing.connection import Connection, wait as multi_wait
from multiprocessing.queues import Queue
from typing import Union
from random import SystemRandom
from core.meow import BasePattern, BaseRecipe, BaseRule
from core.correctness.validation import check_input, valid_dict, valid_list
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
from patterns import *
from recipes import *
from rules import *
from core.correctness.validation import check_type, valid_dict, valid_list
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
VALID_CHANNELS
def check_pattern_dict(patterns, min_length=1):
valid_dict(patterns, str, BasePattern, strict=False, min_length=min_length)
@ -42,8 +42,8 @@ def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
recipes:Union[dict[str,BaseRecipe],list[BaseRecipe]],
new_rules:list[BaseRule]=[])->dict[str,BaseRule]:
check_input(patterns, dict, alt_types=[list])
check_input(recipes, dict, alt_types=[list])
check_type(patterns, dict, alt_types=[list])
check_type(recipes, dict, alt_types=[list])
valid_list(new_rules, BaseRule, min_length=0)
if isinstance(patterns, list):
@ -58,8 +58,9 @@ def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
else:
check_recipe_dict(recipes, min_length=0)
# Imported here to avoid circular imports at top of file
import rules
rules = {}
all_rules ={(r.pattern_type, r.recipe_type):r for r in [r[1] \
for r in inspect.getmembers(sys.modules["rules"], inspect.isclass) \
if (issubclass(r[1], BaseRule))]}
@ -75,4 +76,14 @@ def create_rules(patterns:Union[dict[str,BasePattern],list[BasePattern]],
recipes[pattern.recipe]
)
rules[rule.name] = rule
return rules
return rules
def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]:
all_connections = [i for i in inputs if type(i) is Connection] \
+ [i._reader for i in inputs if type(i) is Queue]
ready = multi_wait(all_connections)
ready_inputs = [i for i in inputs if \
(type(i) is Connection and i in ready) \
or (type(i) is Queue and i._reader in ready)]
return ready_inputs

123
core/meow.py
View File

@ -1,11 +1,11 @@
from multiprocessing.connection import Connection
from typing import Any
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, \
get_not_imp_msg, get_drt_imp_msg
from core.correctness.validation import valid_string
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
get_drt_imp_msg
from core.correctness.validation import valid_string, check_type, \
check_implementation
class BaseRecipe:
@ -15,15 +15,9 @@ class BaseRecipe:
requirements:dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={},
requirements:dict[str,Any]={}):
if (type(self)._is_valid_recipe == BaseRecipe._is_valid_recipe):
msg = get_not_imp_msg(BaseRecipe, BaseRecipe._is_valid_recipe)
raise NotImplementedError(msg)
if (type(self)._is_valid_parameters == BaseRecipe._is_valid_parameters):
msg = get_not_imp_msg(BaseRecipe, BaseRecipe._is_valid_parameters)
raise NotImplementedError(msg)
if (type(self)._is_valid_requirements == BaseRecipe._is_valid_requirements):
msg = get_not_imp_msg(BaseRecipe, BaseRecipe._is_valid_requirements)
raise NotImplementedError(msg)
check_implementation(type(self)._is_valid_recipe, BaseRecipe)
check_implementation(type(self)._is_valid_parameters, BaseRecipe)
check_implementation(type(self)._is_valid_requirements, BaseRecipe)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
@ -55,19 +49,13 @@ class BaseRecipe:
class BasePattern:
name:str
recipe:str
parameters:dict[str, Any]
outputs:dict[str, Any]
parameters:dict[str,Any]
outputs:dict[str,Any]
def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={},
outputs:dict[str,Any]={}):
if (type(self)._is_valid_recipe == BasePattern._is_valid_recipe):
msg = get_not_imp_msg(BasePattern, BasePattern._is_valid_recipe)
raise NotImplementedError(msg)
if (type(self)._is_valid_parameters == BasePattern._is_valid_parameters):
msg = get_not_imp_msg(BasePattern, BasePattern._is_valid_parameters)
raise NotImplementedError(msg)
if (type(self)._is_valid_output == BasePattern._is_valid_output):
msg = get_not_imp_msg(BasePattern, BasePattern._is_valid_output)
raise NotImplementedError(msg)
check_implementation(type(self)._is_valid_recipe, BasePattern)
check_implementation(type(self)._is_valid_parameters, BasePattern)
check_implementation(type(self)._is_valid_output, BasePattern)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
@ -103,13 +91,8 @@ class BaseRule:
pattern_type:str=""
recipe_type:str=""
def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
if (type(self)._is_valid_pattern == BaseRule._is_valid_pattern):
msg = get_not_imp_msg(BaseRule, BaseRule._is_valid_pattern)
raise NotImplementedError(msg)
if (type(self)._is_valid_recipe == BaseRule._is_valid_recipe):
msg = get_not_imp_msg(BaseRule, BaseRule._is_valid_recipe)
raise NotImplementedError(msg)
check_implementation(type(self)._is_valid_pattern, BaseRule)
check_implementation(type(self)._is_valid_recipe, BaseRule)
self._is_valid_name(name)
self.name = name
self._is_valid_pattern(pattern)
@ -144,29 +127,15 @@ class BaseRule:
class BaseMonitor:
rules: dict[str, BaseRule]
report: Connection
listen: Connection
def __init__(self, rules:dict[str, BaseRule], report:Connection,
listen:Connection) -> None:
if (type(self).start == BaseMonitor.start):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor.start)
raise NotImplementedError(msg)
if (type(self).stop == BaseMonitor.stop):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor.stop)
raise NotImplementedError(msg)
if (type(self)._is_valid_report == BaseMonitor._is_valid_report):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor._is_valid_report)
raise NotImplementedError(msg)
report: VALID_CHANNELS
def __init__(self, rules:dict[str,BaseRule],
report:VALID_CHANNELS)->None:
check_implementation(type(self).start, BaseMonitor)
check_implementation(type(self).stop, BaseMonitor)
check_implementation(type(self)._is_valid_report, BaseMonitor)
check_implementation(type(self)._is_valid_rules, BaseMonitor)
self._is_valid_report(report)
self.report = report
if (type(self)._is_valid_listen == BaseMonitor._is_valid_listen):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor._is_valid_listen)
raise NotImplementedError(msg)
self._is_valid_listen(listen)
self.listen = listen
if (type(self)._is_valid_rules == BaseMonitor._is_valid_rules):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor._is_valid_rules)
raise NotImplementedError(msg)
self._is_valid_rules(rules)
self.rules = rules
@ -176,13 +145,10 @@ class BaseMonitor:
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_report(self, report:Connection)->None:
def _is_valid_report(self, report:VALID_CHANNELS)->None:
pass
def _is_valid_listen(self, listen:Connection)->None:
pass
def _is_valid_rules(self, rules:dict[str, BaseRule])->None:
def _is_valid_rules(self, rules:dict[str,BaseRule])->None:
pass
def start(self)->None:
@ -194,13 +160,11 @@ class BaseMonitor:
class BaseHandler:
inputs:Any
def __init__(self, inputs:Any) -> None:
if (type(self).handle == BaseHandler.handle):
msg = get_not_imp_msg(BaseHandler, BaseHandler.handle)
raise NotImplementedError(msg)
if (type(self)._is_valid_inputs == BaseHandler._is_valid_inputs):
msg = get_not_imp_msg(BaseHandler, BaseHandler._is_valid_inputs)
raise NotImplementedError(msg)
def __init__(self, inputs:list[VALID_CHANNELS]) -> None:
check_implementation(type(self).start, BaseHandler)
check_implementation(type(self).stop, BaseHandler)
check_implementation(type(self).handle, BaseHandler)
check_implementation(type(self)._is_valid_inputs, BaseHandler)
self._is_valid_inputs(inputs)
self.inputs = inputs
@ -213,5 +177,34 @@ class BaseHandler:
def _is_valid_inputs(self, inputs:Any)->None:
pass
def handle()->None:
def handle(self, event:Any, rule:BaseRule)->None:
pass
def start(self)->None:
pass
def stop(self)->None:
pass
# TODO test me
class MeowRunner:
monitor:BaseMonitor
handler:BaseHandler
def __init__(self, monitor:BaseMonitor, handler:BaseHandler) -> None:
self._is_valid_monitor(monitor)
self.monitor = monitor
self._is_valid_handler(handler)
self.handler = handler
def start(self)->None:
self.monitor.start()
def stop(self)->None:
self.monitor.stop()
def _is_valid_monitor(self, monitor:BaseMonitor)->None:
check_type(monitor, BaseMonitor)
def _is_valid_handler(self, handler:BaseHandler)->None:
check_type(handler, BaseHandler)