diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index 074d5d9..b5be437 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -12,11 +12,12 @@ from datetime import datetime from typing import Any, Tuple, Dict from core.base_conductor import BaseConductor +from core.correctness.meow import valid_job from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \ STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \ JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR -from core.correctness.validation import valid_job, valid_dir_path +from core.correctness.validation import valid_dir_path from functionality.file_io import make_dir, read_yaml, write_file, write_yaml class LocalPythonConductor(BaseConductor): diff --git a/core/base_rule.py b/core/base_rule.py index 6119f56..ac4eb39 100644 --- a/core/base_rule.py +++ b/core/base_rule.py @@ -6,10 +6,13 @@ for all rule instances. Author(s): David Marchant """ +from sys import modules from typing import Any -from core.base_pattern import BasePattern -from core.base_recipe import BaseRecipe +if "BasePattern" not in modules: + from core.base_pattern import BasePattern +if "BaseRecipe" not in modules: + from core.base_recipe import BaseRecipe from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS from core.correctness.validation import valid_string, check_type, \ check_implementation diff --git a/core/correctness/meow.py b/core/correctness/meow.py new file mode 100644 index 0000000..8acf8fd --- /dev/null +++ b/core/correctness/meow.py @@ -0,0 +1,59 @@ + +from datetime import datetime +from typing import Any, Dict, Type + +from core.base_rule import BaseRule +from core.correctness.validation import check_type +from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \ + JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \ + JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH + +# Required keys in event dict +EVENT_KEYS = { + EVENT_TYPE: str, + EVENT_PATH: str, + # Should be a Rule but can't import here due to circular dependencies + EVENT_RULE: BaseRule +} + +WATCHDOG_EVENT_KEYS = { + WATCHDOG_BASE: str, + WATCHDOG_HASH: str, + **EVENT_KEYS +} + +# Required keys in job dict +JOB_KEYS = { + JOB_TYPE: str, + JOB_EVENT: Dict, + JOB_ID: str, + JOB_PATTERN: Any, + JOB_RECIPE: Any, + JOB_RULE: str, + JOB_STATUS: str, + JOB_CREATE_TIME: datetime, +} + + +def valid_meow_dict(meow_dict:Dict[str,Any], msg:str, + keys:Dict[str,Type])->None: + """Check given dictionary expresses a meow construct. This won't do much + directly, but is called by more specific validation functions.""" + check_type(meow_dict, Dict) + # Check we have all the required keys, and they are all of the expected + # type + for key, value_type in keys.items(): + if not key in meow_dict.keys(): + raise KeyError(f"{msg} require key '{key}'") + check_type(meow_dict[key], value_type) + +def valid_event(event:Dict[str,Any])->None: + """Check that a given dict expresses a meow event.""" + valid_meow_dict(event, "Event", EVENT_KEYS) + +def valid_job(job:Dict[str,Any])->None: + """Check that a given dict expresses a meow job.""" + valid_meow_dict(job, "Job", JOB_KEYS) + +def valid_watchdog_event(event:Dict[str,Any])->None: + valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS) diff --git a/core/correctness/validation.py b/core/correctness/validation.py index b54d19a..846363d 100644 --- a/core/correctness/validation.py +++ b/core/correctness/validation.py @@ -5,43 +5,13 @@ package. Author(s): David Marchant """ -from datetime import datetime + from inspect import signature from os.path import sep, exists, isfile, isdir, dirname from typing import Any, _SpecialForm, Union, Type, Dict, List, \ get_origin, get_args -from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \ - EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \ - JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \ - WATCHDOG_BASE, WATCHDOG_HASH - -# Required keys in event dict -EVENT_KEYS = { - EVENT_TYPE: str, - EVENT_PATH: str, - # TODO sort this - # Should be a Rule but can't import here due to circular dependencies - EVENT_RULE: Any -} - -WATCHDOG_EVENT_KEYS = { - WATCHDOG_BASE: str, - WATCHDOG_HASH: str, - **EVENT_KEYS -} - -# Required keys in job dict -JOB_KEYS = { - JOB_TYPE: str, - JOB_EVENT: Dict, - JOB_ID: str, - JOB_PATTERN: Any, - JOB_RECIPE: Any, - JOB_RULE: str, - JOB_STATUS: str, - JOB_CREATE_TIME: datetime, -} +from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[], or_none:bool=False, hint:str="")->None: @@ -251,26 +221,3 @@ def valid_non_existing_path(variable:str, allow_base:bool=False): if dirname(variable) and not exists(dirname(variable)): raise ValueError( f"Route to requested path '{variable}' does not exist.") - -def valid_meow_dict(meow_dict:Dict[str,Any], msg:str, - keys:Dict[str,Type])->None: - """Check given dictionary expresses a meow construct. This won't do much - directly, but is called by more specific validation functions.""" - check_type(meow_dict, Dict) - # Check we have all the required keys, and they are all of the expected - # type - for key, value_type in keys.items(): - if not key in meow_dict.keys(): - raise KeyError(f"{msg} require key '{key}'") - check_type(meow_dict[key], value_type) - -def valid_event(event:Dict[str,Any])->None: - """Check that a given dict expresses a meow event.""" - valid_meow_dict(event, "Event", EVENT_KEYS) - -def valid_job(job:Dict[str,Any])->None: - """Check that a given dict expresses a meow job.""" - valid_meow_dict(job, "Job", JOB_KEYS) - -def valid_watchdog_event(event:Dict[str,Any])->None: - valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS) diff --git a/functionality/meow.py b/functionality/meow.py index feb6f0f..913e7cf 100644 --- a/functionality/meow.py +++ b/functionality/meow.py @@ -18,7 +18,6 @@ from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH from functionality.naming import generate_job_id, generate_rule_id - # mig trigger keyword replacements KEYWORD_PATH = "{PATH}" KEYWORD_REL_PATH = "{REL_PATH}" diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index e7dc080..a154b2c 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -13,8 +13,9 @@ from typing import Any, Tuple, Dict from core.base_recipe import BaseRecipe from core.base_handler import BaseHandler +from core.correctness.meow import valid_event from core.correctness.validation import check_type, valid_string, \ - valid_dict, valid_path, valid_dir_path, valid_event + valid_dict, valid_path, valid_dir_path from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \ EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \ diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index ed8b52b..4ed3433 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -12,8 +12,9 @@ from typing import Any, Tuple, Dict, List from core.base_recipe import BaseRecipe from core.base_handler import BaseHandler +from core.correctness.meow import valid_event from core.correctness.validation import check_script, valid_string, \ - valid_dict, valid_event, valid_dir_path + valid_dict, valid_dir_path from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \ EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \ diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 9b646a0..9125500 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -1,4 +1,5 @@ +import io import json import unittest import os @@ -15,6 +16,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ PYTHON_FUNC, JOB_ID, JOB_EVENT, \ JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL +from functionality.debug import setup_debugging from functionality.file_io import lines_to_string, make_dir, read_file, \ read_file_lines, read_notebook, read_yaml, rmtree, write_file, \ write_notebook, write_yaml @@ -43,6 +45,21 @@ class DebugTests(unittest.TestCase): super().tearDown() teardown() + # Test setup_debugging will create writeable location + def testSetupDebugging(self)->None: + stream = io.StringIO("") + + target, level = setup_debugging(stream, 1) + + self.assertIsInstance(target, io.StringIO) + self.assertIsInstance(level, int) + + with self.assertRaises(TypeError): + setup_debugging("stream", 1) + + with self.assertRaises(TypeError): + setup_debugging(stream, "1") + class FileIoTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_recipes.py b/tests/test_recipes.py index cc05cc0..06f2257 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -6,13 +6,13 @@ import unittest from multiprocessing import Pipe from typing import Dict +from core.correctness.meow import valid_job from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \ JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \ META_FILE, JOB_ERROR, \ PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \ get_base_file, get_job_file, get_result_file -from core.correctness.validation import valid_job from functionality.file_io import lines_to_string, make_dir, read_yaml, \ write_file, write_notebook, write_yaml from functionality.hashing import get_file_hash diff --git a/tests/test_validation.py b/tests/test_validation.py index b2cd0d7..c58ad83 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,24 +1,24 @@ -import io import unittest import os from datetime import datetime from typing import Any, Union +from core.correctness.meow import valid_event, valid_job, valid_watchdog_event from core.correctness.validation import check_type, check_implementation, \ valid_string, valid_dict, valid_list, valid_existing_file_path, \ - valid_dir_path, valid_non_existing_path, valid_event, valid_job, \ - valid_watchdog_event, check_callable + valid_dir_path, valid_non_existing_path, check_callable from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \ EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \ JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \ WATCHDOG_HASH -from functionality.debug import setup_debugging from functionality.file_io import make_dir -from shared import setup, teardown, TEST_MONITOR_BASE +from functionality.meow import create_rule +from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \ + valid_recipe_one -class CorrectnessTests(unittest.TestCase): +class ValidationTests(unittest.TestCase): def setUp(self)->None: super().setUp() setup() @@ -246,18 +246,35 @@ class CorrectnessTests(unittest.TestCase): with self.assertRaises(ValueError): valid_non_existing_path(test_path) - # TODO modify so tests for actual rule values + # Test check_callable + def testCheckCallable(self)->None: + check_callable(make_dir) + + with self.assertRaises(TypeError): + check_callable("a") + +class MeowTests(unittest.TestCase): + def setUp(self)->None: + super().setUp() + setup() + + def tearDown(self)->None: + super().tearDown() + teardown() + # Test valid_event can check given event dictionary def testEventValidation(self)->None: + rule = create_rule(valid_pattern_one, valid_recipe_one) + valid_event({ EVENT_TYPE: "test", EVENT_PATH: "path", - EVENT_RULE: "rule" + EVENT_RULE: rule }) valid_event({ EVENT_TYPE: "anything", EVENT_PATH: "path", - EVENT_RULE: "rule", + EVENT_RULE: rule, "a": 1 }) @@ -292,27 +309,14 @@ class CorrectnessTests(unittest.TestCase): with self.assertRaises(KeyError): valid_job({}) - # Test setup_debugging will create writeable location - def testSetupDebugging(self)->None: - stream = io.StringIO("") - - target, level = setup_debugging(stream, 1) - - self.assertIsInstance(target, io.StringIO) - self.assertIsInstance(level, int) - - with self.assertRaises(TypeError): - setup_debugging("stream", 1) - - with self.assertRaises(TypeError): - setup_debugging(stream, "1") - # Test watchdog event dict def testWatchdogEventValidation(self)->None: + rule = create_rule(valid_pattern_one, valid_recipe_one) + valid_watchdog_event({ EVENT_TYPE: "test", EVENT_PATH: "path", - EVENT_RULE: "rule", + EVENT_RULE: rule, WATCHDOG_HASH: "hash", WATCHDOG_BASE: "base" }) @@ -340,10 +344,3 @@ class CorrectnessTests(unittest.TestCase): with self.assertRaises(KeyError): valid_event({}) - - # Test check_callable - def testCheckCallable(self)->None: - check_callable(make_dir) - - with self.assertRaises(TypeError): - check_callable("a")