NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

resolved circular dependencies in validation by splitting meow off too

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-10 18:40:15 +01:00
parent 89a0700e1d
commit 9b744e9afe
10 changed files with 119 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
conductors/local_python_conductor.py
View File

@ -12,11 +12,12 @@ from datetime import datetime
from typing import Any, Tuple, Dict
from core.base_conductor import BaseConductor
from core.correctness.meow import valid_job
from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
from core.correctness.validation import valid_job, valid_dir_path
from core.correctness.validation import valid_dir_path
from functionality.file_io import make_dir, read_yaml, write_file, write_yaml
class LocalPythonConductor(BaseConductor):

3
core/base_rule.py
View File

@ -6,9 +6,12 @@ for all rule instances.
Author(s): David Marchant
"""
from sys import modules
from typing import Any
if "BasePattern" not in modules:
from core.base_pattern import BasePattern
if "BaseRecipe" not in modules:
from core.base_recipe import BaseRecipe
from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
from core.correctness.validation import valid_string, check_type, \

59
core/correctness/meow.py Normal file
View File

@ -0,0 +1,59 @@
from datetime import datetime
from typing import Any, Dict, Type
from core.base_rule import BaseRule
from core.correctness.validation import check_type
from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \
JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
# Required keys in event dict
EVENT_KEYS = {
EVENT_TYPE: str,
EVENT_PATH: str,
# Should be a Rule but can't import here due to circular dependencies
EVENT_RULE: BaseRule
}
WATCHDOG_EVENT_KEYS = {
WATCHDOG_BASE: str,
WATCHDOG_HASH: str,
**EVENT_KEYS
}
# Required keys in job dict
JOB_KEYS = {
JOB_TYPE: str,
JOB_EVENT: Dict,
JOB_ID: str,
JOB_PATTERN: Any,
JOB_RECIPE: Any,
JOB_RULE: str,
JOB_STATUS: str,
JOB_CREATE_TIME: datetime,
}
def valid_meow_dict(meow_dict:Dict[str,Any], msg:str,
keys:Dict[str,Type])->None:
"""Check given dictionary expresses a meow construct. This won't do much
directly, but is called by more specific validation functions."""
check_type(meow_dict, Dict)
# Check we have all the required keys, and they are all of the expected
# type
for key, value_type in keys.items():
if not key in meow_dict.keys():
raise KeyError(f"{msg} require key '{key}'")
check_type(meow_dict[key], value_type)
def valid_event(event:Dict[str,Any])->None:
"""Check that a given dict expresses a meow event."""
valid_meow_dict(event, "Event", EVENT_KEYS)
def valid_job(job:Dict[str,Any])->None:
"""Check that a given dict expresses a meow job."""
valid_meow_dict(job, "Job", JOB_KEYS)
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

57
core/correctness/validation.py
View File

@ -5,43 +5,13 @@ package.
Author(s): David Marchant
"""
from datetime import datetime
from inspect import signature
from os.path import sep, exists, isfile, isdir, dirname
from typing import Any, _SpecialForm, Union, Type, Dict, List, \
get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
WATCHDOG_BASE, WATCHDOG_HASH
# Required keys in event dict
EVENT_KEYS = {
EVENT_TYPE: str,
EVENT_PATH: str,
# TODO sort this
# Should be a Rule but can't import here due to circular dependencies
EVENT_RULE: Any
}
WATCHDOG_EVENT_KEYS = {
WATCHDOG_BASE: str,
WATCHDOG_HASH: str,
**EVENT_KEYS
}
# Required keys in job dict
JOB_KEYS = {
JOB_TYPE: str,
JOB_EVENT: Dict,
JOB_ID: str,
JOB_PATTERN: Any,
JOB_RECIPE: Any,
JOB_RULE: str,
JOB_STATUS: str,
JOB_CREATE_TIME: datetime,
}
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
or_none:bool=False, hint:str="")->None:
@ -251,26 +221,3 @@ def valid_non_existing_path(variable:str, allow_base:bool=False):
if dirname(variable) and not exists(dirname(variable)):
raise ValueError(
f"Route to requested path '{variable}' does not exist.")
def valid_meow_dict(meow_dict:Dict[str,Any], msg:str,
keys:Dict[str,Type])->None:
"""Check given dictionary expresses a meow construct. This won't do much
directly, but is called by more specific validation functions."""
check_type(meow_dict, Dict)
# Check we have all the required keys, and they are all of the expected
# type
for key, value_type in keys.items():
if not key in meow_dict.keys():
raise KeyError(f"{msg} require key '{key}'")
check_type(meow_dict[key], value_type)
def valid_event(event:Dict[str,Any])->None:
"""Check that a given dict expresses a meow event."""
valid_meow_dict(event, "Event", EVENT_KEYS)
def valid_job(job:Dict[str,Any])->None:
"""Check that a given dict expresses a meow job."""
valid_meow_dict(job, "Job", JOB_KEYS)
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

1
functionality/meow.py
View File

@ -18,7 +18,6 @@ from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH
from functionality.naming import generate_job_id, generate_rule_id
# mig trigger keyword replacements
KEYWORD_PATH = "{PATH}"
KEYWORD_REL_PATH = "{REL_PATH}"

3
recipes/jupyter_notebook_recipe.py
View File

@ -13,8 +13,9 @@ from typing import Any, Tuple, Dict
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.meow import valid_event
from core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path, valid_event
valid_dict, valid_path, valid_dir_path
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \

3
recipes/python_recipe.py
View File

@ -12,8 +12,9 @@ from typing import Any, Tuple, Dict, List
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.meow import valid_event
from core.correctness.validation import check_script, valid_string, \
valid_dict, valid_event, valid_dir_path
valid_dict, valid_dir_path
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \

17
tests/test_functionality.py
View File

@ -1,4 +1,5 @@
import io
import json
import unittest
import os
@ -15,6 +16,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from functionality.debug import setup_debugging
from functionality.file_io import lines_to_string, make_dir, read_file, \
read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
write_notebook, write_yaml
@ -43,6 +45,21 @@ class DebugTests(unittest.TestCase):
super().tearDown()
teardown()
# Test setup_debugging will create writeable location
def testSetupDebugging(self)->None:
stream = io.StringIO("")
target, level = setup_debugging(stream, 1)
self.assertIsInstance(target, io.StringIO)
self.assertIsInstance(level, int)
with self.assertRaises(TypeError):
setup_debugging("stream", 1)
with self.assertRaises(TypeError):
setup_debugging(stream, "1")
class FileIoTests(unittest.TestCase):
def setUp(self)->None:

2
tests/test_recipes.py
View File

@ -6,13 +6,13 @@ import unittest
from multiprocessing import Pipe
from typing import Dict
from core.correctness.meow import valid_job
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
META_FILE, JOB_ERROR, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from core.correctness.validation import valid_job
from functionality.file_io import lines_to_string, make_dir, read_yaml, \
write_file, write_notebook, write_yaml
from functionality.hashing import get_file_hash

61
tests/test_validation.py
View File

@ -1,24 +1,24 @@
import io
import unittest
import os
from datetime import datetime
from typing import Any, Union
from core.correctness.meow import valid_event, valid_job, valid_watchdog_event
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_dir_path, valid_non_existing_path, valid_event, valid_job, \
valid_watchdog_event, check_callable
valid_dir_path, valid_non_existing_path, check_callable
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
WATCHDOG_HASH
from functionality.debug import setup_debugging
from functionality.file_io import make_dir
from shared import setup, teardown, TEST_MONITOR_BASE
from functionality.meow import create_rule
from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \
valid_recipe_one
class CorrectnessTests(unittest.TestCase):
class ValidationTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
@ -246,18 +246,35 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(ValueError):
valid_non_existing_path(test_path)
# TODO modify so tests for actual rule values
# Test check_callable
def testCheckCallable(self)->None:
check_callable(make_dir)
with self.assertRaises(TypeError):
check_callable("a")
class MeowTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test valid_event can check given event dictionary
def testEventValidation(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
valid_event({
EVENT_TYPE: "test",
EVENT_PATH: "path",
EVENT_RULE: "rule"
EVENT_RULE: rule
})
valid_event({
EVENT_TYPE: "anything",
EVENT_PATH: "path",
EVENT_RULE: "rule",
EVENT_RULE: rule,
"a": 1
})
@ -292,27 +309,14 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(KeyError):
valid_job({})
# Test setup_debugging will create writeable location
def testSetupDebugging(self)->None:
stream = io.StringIO("")
target, level = setup_debugging(stream, 1)
self.assertIsInstance(target, io.StringIO)
self.assertIsInstance(level, int)
with self.assertRaises(TypeError):
setup_debugging("stream", 1)
with self.assertRaises(TypeError):
setup_debugging(stream, "1")
# Test watchdog event dict
def testWatchdogEventValidation(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
valid_watchdog_event({
EVENT_TYPE: "test",
EVENT_PATH: "path",
EVENT_RULE: "rule",
EVENT_RULE: rule,
WATCHDOG_HASH: "hash",
WATCHDOG_BASE: "base"
})
@ -340,10 +344,3 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(KeyError):
valid_event({})
# Test check_callable
def testCheckCallable(self)->None:
check_callable(make_dir)
with self.assertRaises(TypeError):
check_callable("a")