resolved circular dependencies in validation by splitting meow off too

conductors/local_python_conductor.py

@@ -12,11 +12,12 @@ from datetime import datetime
 from typing import Any, Tuple, Dict
 
 from core.base_conductor import BaseConductor
+from core.correctness.meow import valid_job
 from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
     STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
     STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
     JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
-from core.correctness.validation import valid_job, valid_dir_path
+from core.correctness.validation import valid_dir_path
 from functionality.file_io import make_dir, read_yaml, write_file, write_yaml
 
 class LocalPythonConductor(BaseConductor):

core/base_rule.py

@@ -6,9 +6,12 @@ for all rule instances.
 Author(s): David Marchant
 """
 
+from sys import modules
 from typing import Any
 
+if "BasePattern" not in modules:
     from core.base_pattern import BasePattern
+if "BaseRecipe" not in modules:
     from core.base_recipe import BaseRecipe
 from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
 from core.correctness.validation import valid_string, check_type, \

core/correctness/meow.py

@@ -0,0 +1,59 @@
+
+from datetime import datetime
+from typing import Any, Dict, Type
+
+from core.base_rule import BaseRule
+from core.correctness.validation import check_type
+from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
+    JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \
+    JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
+
+# Required keys in event dict
+EVENT_KEYS = {
+    EVENT_TYPE: str,
+    EVENT_PATH: str,
+    # Should be a Rule but can't import here due to circular dependencies
+    EVENT_RULE: BaseRule
+}
+
+WATCHDOG_EVENT_KEYS = {
+    WATCHDOG_BASE: str,
+    WATCHDOG_HASH: str,    
+    **EVENT_KEYS
+}
+
+# Required keys in job dict
+JOB_KEYS = {
+    JOB_TYPE: str,
+    JOB_EVENT: Dict,
+    JOB_ID: str,
+    JOB_PATTERN: Any,
+    JOB_RECIPE: Any,
+    JOB_RULE: str,
+    JOB_STATUS: str,
+    JOB_CREATE_TIME: datetime,
+}
+
+
+def valid_meow_dict(meow_dict:Dict[str,Any], msg:str, 
+        keys:Dict[str,Type])->None:
+    """Check given dictionary expresses a meow construct. This won't do much 
+    directly, but is called by more specific validation functions."""
+    check_type(meow_dict, Dict)
+    # Check we have all the required keys, and they are all of the expected 
+    # type
+    for key, value_type in keys.items():
+        if not key in meow_dict.keys():
+            raise KeyError(f"{msg} require key '{key}'")
+        check_type(meow_dict[key], value_type)
+
+def valid_event(event:Dict[str,Any])->None:
+    """Check that a given dict expresses a meow event."""
+    valid_meow_dict(event, "Event", EVENT_KEYS)
+
+def valid_job(job:Dict[str,Any])->None:
+    """Check that a given dict expresses a meow job."""
+    valid_meow_dict(job, "Job", JOB_KEYS)
+
+def valid_watchdog_event(event:Dict[str,Any])->None:
+    valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

core/correctness/validation.py

@@ -5,43 +5,13 @@ package.
 
 Author(s): David Marchant
 """
-from datetime import datetime
+
 from inspect import signature
 from os.path import sep, exists, isfile, isdir, dirname
 from typing import Any, _SpecialForm, Union, Type, Dict, List, \
     get_origin, get_args
 
-from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \
+from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
-    EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
-    JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
-    WATCHDOG_BASE, WATCHDOG_HASH
-
-# Required keys in event dict
-EVENT_KEYS = {
-    EVENT_TYPE: str,
-    EVENT_PATH: str,
-    # TODO sort this
-    # Should be a Rule but can't import here due to circular dependencies
-    EVENT_RULE: Any
-}
-
-WATCHDOG_EVENT_KEYS = {
-    WATCHDOG_BASE: str,
-    WATCHDOG_HASH: str,    
-    **EVENT_KEYS
-}
-
-# Required keys in job dict
-JOB_KEYS = {
-    JOB_TYPE: str,
-    JOB_EVENT: Dict,
-    JOB_ID: str,
-    JOB_PATTERN: Any,
-    JOB_RECIPE: Any,
-    JOB_RULE: str,
-    JOB_STATUS: str,
-    JOB_CREATE_TIME: datetime,
-}
 
 def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[], 
         or_none:bool=False, hint:str="")->None:
@@ -251,26 +221,3 @@ def valid_non_existing_path(variable:str, allow_base:bool=False):
     if dirname(variable) and not exists(dirname(variable)):
         raise ValueError(
             f"Route to requested path '{variable}' does not exist.")
-
-def valid_meow_dict(meow_dict:Dict[str,Any], msg:str, 
-        keys:Dict[str,Type])->None:
-    """Check given dictionary expresses a meow construct. This won't do much 
-    directly, but is called by more specific validation functions."""
-    check_type(meow_dict, Dict)
-    # Check we have all the required keys, and they are all of the expected 
-    # type
-    for key, value_type in keys.items():
-        if not key in meow_dict.keys():
-            raise KeyError(f"{msg} require key '{key}'")
-        check_type(meow_dict[key], value_type)
-
-def valid_event(event:Dict[str,Any])->None:
-    """Check that a given dict expresses a meow event."""
-    valid_meow_dict(event, "Event", EVENT_KEYS)
-
-def valid_job(job:Dict[str,Any])->None:
-    """Check that a given dict expresses a meow job."""
-    valid_meow_dict(job, "Job", JOB_KEYS)
-
-def valid_watchdog_event(event:Dict[str,Any])->None:
-    valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

functionality/meow.py

@@ -18,7 +18,6 @@ from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
     STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH
 from functionality.naming import generate_job_id, generate_rule_id
 
-
 # mig trigger keyword replacements
 KEYWORD_PATH = "{PATH}"
 KEYWORD_REL_PATH = "{REL_PATH}"

recipes/jupyter_notebook_recipe.py

@@ -13,8 +13,9 @@ from typing import Any, Tuple, Dict
 
 from core.base_recipe import BaseRecipe
 from core.base_handler import BaseHandler
+from core.correctness.meow import valid_event
 from core.correctness.validation import check_type, valid_string, \
-    valid_dict, valid_path, valid_dir_path, valid_event
+    valid_dict, valid_path, valid_dir_path
 from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
     EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \

recipes/python_recipe.py

@@ -12,8 +12,9 @@ from typing import Any, Tuple, Dict, List
 
 from core.base_recipe import BaseRecipe
 from core.base_handler import BaseHandler
+from core.correctness.meow import valid_event
 from core.correctness.validation import check_script, valid_string, \
-    valid_dict, valid_event, valid_dir_path
+    valid_dict, valid_dir_path
 from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
     EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \

tests/test_functionality.py

@@ -1,4 +1,5 @@
 
+import io
 import json
 import unittest
 import os
@@ -15,6 +16,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
     PYTHON_FUNC, JOB_ID, JOB_EVENT, \
     JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
     JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
+from functionality.debug import setup_debugging
 from functionality.file_io import lines_to_string, make_dir, read_file, \
     read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
     write_notebook, write_yaml    
@@ -43,6 +45,21 @@ class DebugTests(unittest.TestCase):
         super().tearDown()
         teardown()
 
+    # Test setup_debugging will create writeable location
+    def testSetupDebugging(self)->None:
+        stream = io.StringIO("")
+
+        target, level = setup_debugging(stream, 1)
+
+        self.assertIsInstance(target, io.StringIO)
+        self.assertIsInstance(level, int)
+
+        with self.assertRaises(TypeError):
+            setup_debugging("stream", 1)
+
+        with self.assertRaises(TypeError):
+            setup_debugging(stream, "1")
+
 
 class FileIoTests(unittest.TestCase):
     def setUp(self)->None:

tests/test_recipes.py

@@ -6,13 +6,13 @@ import unittest
 from multiprocessing import Pipe
 from typing import Dict
 
+from core.correctness.meow import valid_job
 from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
     EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
     JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
     META_FILE, JOB_ERROR, \
     PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
     get_base_file, get_job_file, get_result_file
-from core.correctness.validation import valid_job
 from functionality.file_io import lines_to_string, make_dir, read_yaml, \
     write_file, write_notebook, write_yaml
 from functionality.hashing import get_file_hash

tests/test_validation.py

@@ -1,24 +1,24 @@
 
-import io
 import unittest
 import os
 
 from datetime import datetime
 from typing import Any, Union
 
+from core.correctness.meow import valid_event, valid_job, valid_watchdog_event
 from core.correctness.validation import check_type, check_implementation, \
     valid_string, valid_dict, valid_list, valid_existing_file_path, \
-    valid_dir_path, valid_non_existing_path, valid_event, valid_job, \
+    valid_dir_path, valid_non_existing_path, check_callable
-    valid_watchdog_event, check_callable
 from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
     EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
     JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
     WATCHDOG_HASH
-from functionality.debug import setup_debugging
 from functionality.file_io import make_dir
-from shared import setup, teardown, TEST_MONITOR_BASE
+from functionality.meow import create_rule
+from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \
+    valid_recipe_one
 
-class CorrectnessTests(unittest.TestCase):
+class ValidationTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
         setup()
@@ -246,18 +246,35 @@ class CorrectnessTests(unittest.TestCase):
         with self.assertRaises(ValueError):
             valid_non_existing_path(test_path)
 
-    # TODO modify so tests for actual rule values
+    # Test check_callable 
+    def testCheckCallable(self)->None:
+        check_callable(make_dir)
+        
+        with self.assertRaises(TypeError):
+            check_callable("a")
+
+class MeowTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+        
     # Test valid_event can check given event dictionary
     def testEventValidation(self)->None:
+        rule = create_rule(valid_pattern_one, valid_recipe_one)
+
         valid_event({
             EVENT_TYPE: "test", 
             EVENT_PATH: "path", 
-            EVENT_RULE: "rule"
+            EVENT_RULE: rule
         })
         valid_event({
             EVENT_TYPE: "anything", 
             EVENT_PATH: "path", 
-            EVENT_RULE: "rule",
+            EVENT_RULE: rule,
             "a": 1
         })
 
@@ -292,27 +309,14 @@ class CorrectnessTests(unittest.TestCase):
         with self.assertRaises(KeyError):
             valid_job({})
 
-    # Test setup_debugging will create writeable location
-    def testSetupDebugging(self)->None:
-        stream = io.StringIO("")
-
-        target, level = setup_debugging(stream, 1)
-
-        self.assertIsInstance(target, io.StringIO)
-        self.assertIsInstance(level, int)
-
-        with self.assertRaises(TypeError):
-            setup_debugging("stream", 1)
-
-        with self.assertRaises(TypeError):
-            setup_debugging(stream, "1")
-
     # Test watchdog event dict
     def testWatchdogEventValidation(self)->None:
+        rule = create_rule(valid_pattern_one, valid_recipe_one)
+
         valid_watchdog_event({
             EVENT_TYPE: "test", 
             EVENT_PATH: "path", 
-            EVENT_RULE: "rule",
+            EVENT_RULE: rule,
             WATCHDOG_HASH: "hash",
             WATCHDOG_BASE: "base"
         })
@@ -340,10 +344,3 @@ class CorrectnessTests(unittest.TestCase):
 
         with self.assertRaises(KeyError):
             valid_event({})
-
-    # Test check_callable 
-    def testCheckCallable(self)->None:
-        check_callable(make_dir)
-        
-        with self.assertRaises(TypeError):
-            check_callable("a")