added tests for handlers and conductors validating events and jobs

conductors/local_python_conductor.py

@@ -38,11 +38,15 @@ class LocalPythonConductor(BaseConductor):
         process it or not. This conductor will accept any Python job type"""
         try:
             valid_job(job)
-            if job[JOB_TYPE] in [JOB_TYPE_PYTHON, JOB_TYPE_PAPERMILL]:
+            msg = ""
+            if job[JOB_TYPE] not in [JOB_TYPE_PYTHON, JOB_TYPE_PAPERMILL]:
+                msg = "Job type was not in python or papermill. "
+            if msg:
+                return False, msg
+            else:
                 return True, ""
         except Exception as e:
-            pass
+            return False, str(e)
-        return False, str(e)
 
     def execute(self, job_dir:str)->None:
         """Function to actually execute a Python job. This will read job 

recipes/jupyter_notebook_recipe.py

@@ -112,12 +112,17 @@ class PapermillHandler(BaseHandler):
         jupyter notebook recipes."""
         try:
             valid_event(event)
-            if type(event[EVENT_RULE].recipe) == JupyterNotebookRecipe \
+            msg = ""
-                    and event[EVENT_TYPE] == EVENT_TYPE_WATCHDOG:
+            if type(event[EVENT_RULE].recipe) != JupyterNotebookRecipe: 
+                msg = "Recipe is not a JupyterNotebookRecipe. "
+            if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG:
+                msg += f"Event type is not {EVENT_TYPE_WATCHDOG}."
+            if msg:
+                return False, msg
+            else:
                 return True, ""
         except Exception as e:
-            pass
+            return False, str(e)
-        return False, str(e)
 
     def _is_valid_job_queue_dir(self, job_queue_dir)->None:
         """Validation check for 'job_queue_dir' variable from main 

recipes/python_recipe.py

@@ -101,12 +101,17 @@ class PythonHandler(BaseHandler):
         Python recipes"""
         try:
             valid_event(event)
-            if event[EVENT_TYPE] == EVENT_TYPE_WATCHDOG \
+            msg = ""
-                    and type(event[EVENT_RULE].recipe) == PythonRecipe:
+            if type(event[EVENT_RULE].recipe) != PythonRecipe:
+                msg = "Recipe is not a PythonRecipe. "
+            if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG:
+                msg += f"Event type is not {EVENT_TYPE_WATCHDOG}."
+            if msg:
+                return False, msg
+            else:
                 return True, ""
         except Exception as e:
-            pass
+            return False, str(e)
-        return False, str(e)
 
     def _is_valid_job_queue_dir(self, job_queue_dir)->None:
         """Validation check for 'job_queue_dir' variable from main 

tests/test_conductors.py

@@ -2,13 +2,15 @@
 import os
 import unittest
 
+from datetime import datetime
 from typing import Dict
 
 from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
-    JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, \
+    JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, JOB_EVENT, \
-    META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
+    META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, JOB_PATTERN, \
-    STATUS_DONE, JOB_TYPE_PAPERMILL, get_base_file, get_result_file, \
+    STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
-    get_job_file
+    JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
+    EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
 from core.functionality import get_file_hash, create_watchdog_event, \
     create_job, make_dir, write_yaml, write_notebook, read_yaml, write_file, \
     lines_to_string, read_file
@@ -19,7 +21,8 @@ from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
     papermill_job_func
 from recipes.python_recipe import PythonRecipe, python_job_func
 from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
-    TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT
+    TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, \
+    BAREBONES_PYTHON_SCRIPT, BAREBONES_NOTEBOOK
 
 
 def failing_func():
@@ -622,6 +625,96 @@ class MeowTests(unittest.TestCase):
             "Recieved incorrectly setup job.\n\n\"Job require key "
             "'job_type'\"")
 
+   # Test execute criteria function
+    def testValidExecuteCriteria(self)->None:
+        lpc = LocalPythonConductor()
+
+        pattern_python = FileEventPattern(
+            "pattern_python", "A", "recipe_python", "file_one")
+        recipe_python = PythonRecipe(
+            "recipe_python", BAREBONES_PYTHON_SCRIPT
+        )
+
+        pattern_papermill = FileEventPattern(
+            "pattern_papermill", "A", "recipe_papermill", "file_one")
+        recipe_papermill = JupyterNotebookRecipe(
+            "recipe_papermill", BAREBONES_NOTEBOOK
+        )
+
+        python_rule = create_rule(pattern_python, recipe_python)
+        papermill_rule = create_rule(pattern_papermill, recipe_papermill)
+
+        status, _ = lpc.valid_execute_criteria({})
+        self.assertFalse(status)
+
+        status, _ = lpc.valid_execute_criteria("")
+        self.assertFalse(status)
+
+        status, _ = lpc.valid_execute_criteria({
+            JOB_ID: "path",
+            JOB_EVENT: "type",
+            JOB_TYPE: "rule",
+            JOB_PATTERN: "pattern",
+            JOB_RECIPE: "recipe",
+            JOB_RULE: "rule",
+            JOB_STATUS: "status",
+            JOB_CREATE_TIME: "create",
+            JOB_REQUIREMENTS: "requirements"
+        })
+        self.assertFalse(status)
+
+        status, s = lpc.valid_execute_criteria({
+            JOB_ID: "path",
+            JOB_EVENT: {
+                EVENT_PATH: "path",
+                EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+                EVENT_RULE: python_rule
+            },
+            JOB_TYPE: "type",
+            JOB_PATTERN: python_rule.pattern.name,
+            JOB_RECIPE: python_rule.recipe.name,
+            JOB_RULE: python_rule.name,
+            JOB_STATUS: "status",
+            JOB_CREATE_TIME: datetime.now(),
+            JOB_REQUIREMENTS: python_rule.recipe.requirements
+        })
+        print(s)
+        self.assertFalse(status)
+
+        status, s = lpc.valid_execute_criteria({
+            JOB_ID: "path",
+            JOB_EVENT: {
+                EVENT_PATH: "path",
+                EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+                EVENT_RULE: python_rule
+            },
+            JOB_TYPE: JOB_TYPE_PYTHON,
+            JOB_PATTERN: python_rule.pattern.name,
+            JOB_RECIPE: python_rule.recipe.name,
+            JOB_RULE: python_rule.name,
+            JOB_STATUS: "status",
+            JOB_CREATE_TIME: datetime.now(),
+            JOB_REQUIREMENTS: python_rule.recipe.requirements
+        })
+        print(s)
+        self.assertTrue(status)
+
+        status, s = lpc.valid_execute_criteria({
+            JOB_ID: "path",
+            JOB_EVENT: {
+                EVENT_PATH: "path",
+                EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+                EVENT_RULE: papermill_rule
+            },
+            JOB_TYPE: JOB_TYPE_PYTHON,
+            JOB_PATTERN: papermill_rule.pattern.name,
+            JOB_RECIPE: papermill_rule.recipe.name,
+            JOB_RULE: papermill_rule.name,
+            JOB_STATUS: "status",
+            JOB_CREATE_TIME: datetime.now(),
+            JOB_REQUIREMENTS: papermill_rule.recipe.requirements
+        })
+        print(s)
+        self.assertTrue(status)
+
     # TODO test job status funcs
-    # TODO test mangled status file reads
-    # TODO test missing input files

tests/test_recipes.py

@@ -401,7 +401,44 @@ class JupyterNotebookTests(unittest.TestCase):
         self.assertEqual(len(os.listdir(TEST_JOB_QUEUE)), 0)
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
-    #TODO Test handling criteria function
+    # Test handling criteria function
+    def testValidHandleCriteria(self)->None:
+        ph = PapermillHandler()
+
+        pattern = FileEventPattern(
+            "pattern_ne", "A", "recipe_one", "file_one")
+        recipe = JupyterNotebookRecipe(
+            "recipe_one", COMPLETE_NOTEBOOK)
+
+        rule = create_rule(pattern, recipe)
+
+        status, _ = ph.valid_handle_criteria({})
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria("")
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: "type",
+            EVENT_RULE: rule
+        })
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_RULE: "rule"
+        })
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_RULE: rule
+        })
+        self.assertTrue(status)
+
 
 class PythonTests(unittest.TestCase):
     def setUp(self)->None:
@@ -754,4 +791,44 @@ class PythonTests(unittest.TestCase):
         self.assertEqual(len(os.listdir(TEST_JOB_QUEUE)), 0)
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
+    # Test handling criteria function
+    def testValidHandleCriteria(self)->None:
+        ph = PythonHandler()
+
+        pattern = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one")
+        recipe = PythonRecipe(
+            "recipe_one", BAREBONES_PYTHON_SCRIPT
+        )
+
+        rule = create_rule(pattern, recipe)
+
+        status, _ = ph.valid_handle_criteria({})
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria("")
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: "type",
+            EVENT_RULE: rule
+        })
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_RULE: "rule"
+        })
+        self.assertFalse(status)
+
+        status, s = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_RULE: rule
+        })
+        self.assertTrue(status)
+
+
     # TODO test default parameter function execution

tests/test_runner.py

@@ -430,7 +430,6 @@ class MeowTests(unittest.TestCase):
         runner.stop()
 
         job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
-        print(os.listdir(job_dir))
 
         metafile = os.path.join(job_dir, META_FILE)
         status = read_yaml(metafile)