From 64aaf46196064ed27791dfd631a5572485abf6c3 Mon Sep 17 00:00:00 2001 From: PatchOfScotland Date: Thu, 2 Feb 2023 17:41:18 +0100 Subject: [PATCH] standardised event construction and validation a bit more --- conductors/local_python_conductor.py | 10 +-- core/correctness/validation.py | 11 ++- core/functionality.py | 40 ++++++++-- core/meow.py | 7 +- core/runner.py | 8 +- patterns/file_event_pattern.py | 20 ++--- recipes/jupyter_notebook_recipe.py | 12 +-- recipes/python_recipe.py | 12 +-- tests/test_conductors.py | 52 +++++-------- tests/test_functionality.py | 105 ++++++++++++++++++++++++++- tests/test_meow.py | 8 +- tests/test_recipes.py | 16 ++-- tests/test_validation.py | 39 +++++++++- 13 files changed, 243 insertions(+), 97 deletions(-) diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index 81312c9..7651584 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -9,7 +9,7 @@ import os import shutil from datetime import datetime -from typing import Any +from typing import Any, Tuple from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \ @@ -25,16 +25,16 @@ class LocalPythonConductor(BaseConductor): def __init__(self)->None: super().__init__() - def valid_execute_criteria(self, job:dict[str,Any])->bool: + def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]: """Function to determine given an job defintion, if this conductor can process it or not. This conductor will accept any Python job type""" try: valid_job(job) if job[JOB_TYPE] == JOB_TYPE_PYTHON: - return True - except: + return True, "" + except Exception as e: pass - return False + return False, str(e) def execute(self, job:dict[str,Any])->None: valid_job(job) diff --git a/core/correctness/validation.py b/core/correctness/validation.py index 50b83c3..bca3135 100644 --- a/core/correctness/validation.py +++ b/core/correctness/validation.py @@ -12,7 +12,8 @@ from typing import Any, _SpecialForm, Union, Tuple, get_origin, get_args from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \ EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \ - JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE + JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \ + WATCHDOG_BASE # Required keys in event dict EVENT_KEYS = { @@ -23,6 +24,11 @@ EVENT_KEYS = { EVENT_RULE: Any } +WATCHDOG_EVENT_KEYS = { + WATCHDOG_BASE: str, + **EVENT_KEYS +} + # Required keys in job dict JOB_KEYS = { JOB_TYPE: str, @@ -254,3 +260,6 @@ def valid_event(event:dict[str,Any])->None: def valid_job(job:dict[str,Any])->None: """Check that a given dict expresses a meow job.""" valid_meow_dict(job, "Job", JOB_KEYS) + +def valid_watchdog_event(event:dict[str,Any])->None: + valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS) diff --git a/core/functionality.py b/core/functionality.py index 33b3e86..dd1bcbb 100644 --- a/core/functionality.py +++ b/core/functionality.py @@ -20,7 +20,8 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ VALID_CHANNELS, HASH_BUFFER_SIZE, SHA256, DEBUG_WARNING, DEBUG_INFO, \ EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \ JOB_RECIPE, JOB_RULE, EVENT_RULE, JOB_STATUS, STATUS_QUEUED, \ - JOB_CREATE_TIME, JOB_REQUIREMENTS + JOB_CREATE_TIME, JOB_REQUIREMENTS, WATCHDOG_BASE, WATCHDOG_HASH, \ + EVENT_TYPE_WATCHDOG, JOB_TYPE_PYTHON # mig trigger keyword replacements KEYWORD_PATH = "{PATH}" @@ -283,16 +284,45 @@ def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str, return new_dict -def create_event(event_type:str, path:str, rule:Any, source:dict[Any,Any]={} +def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={} )->dict[Any,Any]: return { - **source, + **extras, EVENT_PATH: path, EVENT_TYPE: event_type, EVENT_RULE: rule } -def create_job(job_type:str, event:dict[str,Any], source:dict[Any,Any]={} +def create_watchdog_event(path:str, rule:Any, base:str, hash:str, + extras:dict[Any,Any]={})->dict[Any,Any]: + return create_event( + EVENT_TYPE_WATCHDOG, + path, + rule, + extras={ + **extras, + **{ + WATCHDOG_HASH: hash, + WATCHDOG_BASE: base + } + } + ) + +def create_fake_watchdog_event(path:str, rule:Any, base:str, + extras:dict[Any,Any]={})->dict[Any,Any]: + return create_event( + EVENT_TYPE_WATCHDOG, + path, + rule, + extras={ + **extras, + **{ + WATCHDOG_BASE: base + } + } + ) + +def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={} )->dict[Any,Any]: job_dict = { #TODO compress event? @@ -307,4 +337,4 @@ def create_job(job_type:str, event:dict[str,Any], source:dict[Any,Any]={} JOB_REQUIREMENTS: event[EVENT_RULE].recipe.requirements } - return {**source, **job_dict} + return {**extras, **job_dict} diff --git a/core/meow.py b/core/meow.py index f4a80b4..498948b 100644 --- a/core/meow.py +++ b/core/meow.py @@ -11,7 +11,7 @@ import inspect import sys from copy import deepcopy -from typing import Any, Union +from typing import Any, Union, Tuple from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \ @@ -323,8 +323,7 @@ class BaseHandler: raise TypeError(msg) return object.__new__(cls) - # TODO also implement something like me from conductor - def valid_handle_criteria(self, event:dict[str,Any])->bool: + def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]: """Function to determine given an event defintion, if this handler can process it or not. Must be implemented by any child process.""" pass @@ -350,7 +349,7 @@ class BaseConductor: raise TypeError(msg) return object.__new__(cls) - def valid_execute_criteria(self, job:dict[str,Any])->bool: + def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]: """Function to determine given an job defintion, if this conductor can process it or not. Must be implemented by any child process.""" pass diff --git a/core/runner.py b/core/runner.py index ed4a2c3..ac5c9f9 100644 --- a/core/runner.py +++ b/core/runner.py @@ -10,13 +10,12 @@ import os import sys import threading -from inspect import signature from multiprocessing import Pipe from random import randrange from typing import Any, Union from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \ - VALID_CHANNELS, JOB_TYPE, JOB_ID, META_FILE + VALID_CHANNELS, JOB_ID, META_FILE from core.correctness.validation import setup_debugging, check_type, \ valid_list from core.functionality import print_debug, wait, read_yaml @@ -104,7 +103,7 @@ class MeowRunner: valid_handlers = [] for handler in self.handlers: try: - valid = handler.valid_handle_criteria(event) + valid, _ = handler.valid_handle_criteria(event) if valid: valid_handlers.append(handler) except Exception as e: @@ -155,7 +154,8 @@ class MeowRunner: valid_conductors = [] for conductor in self.conductors: try: - valid = conductor.valid_execute_criteria(job) + valid, _ = \ + conductor.valid_execute_criteria(job) if valid: valid_conductors.append(conductor) except Exception as e: diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 2136458..0f04a3f 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -25,7 +25,8 @@ from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \ FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \ WATCHDOG_BASE, FILE_RETROACTIVE_EVENT, WATCHDOG_HASH, SHA256 -from core.functionality import print_debug, create_event, get_file_hash +from core.functionality import print_debug, create_watchdog_event, \ + get_file_hash, create_fake_watchdog_event from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \ create_rule @@ -236,17 +237,11 @@ class WatchdogMonitor(BaseMonitor): # If matched, the create a watchdog event if direct_hit or recursive_hit: - meow_event = create_event( - EVENT_TYPE_WATCHDOG, + meow_event = create_watchdog_event( event.src_path, rule, - { - WATCHDOG_BASE: self.base_dir, - WATCHDOG_HASH: get_file_hash( - event.src_path, - SHA256 - ) - } + self.base_dir, + get_file_hash(event.src_path, SHA256) ) print_debug(self._print_target, self.debug_level, f"Event at {src_path} of type {event_type} hit rule " @@ -534,11 +529,10 @@ class WatchdogMonitor(BaseMonitor): # For each file create a fake event. for globble in globbed: - meow_event = create_event( - EVENT_TYPE_WATCHDOG, + meow_event = create_fake_watchdog_event( globble, rule, - { WATCHDOG_BASE: self.base_dir } + self.base_dir ) print_debug(self._print_target, self.debug_level, f"Retroactive event for file at at {globble} hit rule " diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index 6d7ff11..60a8606 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -10,7 +10,7 @@ import itertools import nbformat import sys -from typing import Any +from typing import Any, Tuple from core.correctness.validation import check_type, valid_string, \ valid_dict, valid_path, valid_existing_dir_path, setup_debugging, \ @@ -123,7 +123,7 @@ class PapermillHandler(BaseHandler): yaml_dict[value[0]] = value[1] self.setup_job(event, yaml_dict) - def valid_handle_criteria(self, event:dict[str,Any])->bool: + def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]: """Function to determine given an event defintion, if this handler can process it or not. This handler accepts events from watchdog with jupyter notebook recipes.""" @@ -131,10 +131,10 @@ class PapermillHandler(BaseHandler): valid_event(event) if type(event[EVENT_RULE].recipe) == JupyterNotebookRecipe \ and event[EVENT_TYPE] == EVENT_TYPE_WATCHDOG: - return True - except: + return True, "" + except Exception as e: pass - return False + return False, str(e) def _is_valid_handler_base(self, handler_base)->None: @@ -153,7 +153,7 @@ class PapermillHandler(BaseHandler): meow_job = create_job( JOB_TYPE_PYTHON, event, - { + extras={ JOB_PARAMETERS:yaml_dict, JOB_HASH: event[WATCHDOG_HASH], PYTHON_FUNC:job_func, diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index df71cb4..a5352b5 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -10,7 +10,7 @@ import itertools import nbformat import sys -from typing import Any +from typing import Any, Tuple from core.correctness.validation import check_type, valid_string, \ valid_dict, valid_event, valid_existing_dir_path, setup_debugging @@ -114,7 +114,7 @@ class PythonHandler(BaseHandler): yaml_dict[value[0]] = value[1] self.setup_job(event, yaml_dict) - def valid_handle_criteria(self, event:dict[str,Any])->bool: + def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]: """Function to determine given an event defintion, if this handler can process it or not. This handler accepts events from watchdog with Python recipes""" @@ -122,10 +122,10 @@ class PythonHandler(BaseHandler): valid_event(event) if event[EVENT_TYPE] == EVENT_TYPE_WATCHDOG \ and type(event[EVENT_RULE].recipe) == PythonRecipe: - return True - except: + return True, "" + except Exception as e: pass - return False + return False, str(e) def _is_valid_handler_base(self, handler_base)->None: """Validation check for 'handler_base' variable from main @@ -143,7 +143,7 @@ class PythonHandler(BaseHandler): meow_job = create_job( JOB_TYPE_PYTHON, event, - { + extras={ JOB_PARAMETERS:yaml_dict, JOB_HASH: event[WATCHDOG_HASH], PYTHON_FUNC:job_func, diff --git a/tests/test_conductors.py b/tests/test_conductors.py index 8e2ac2a..7cfcdad 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -6,8 +6,8 @@ from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, EVENT_TYPE_WATCHDOG, WATCHDOG_BASE, EVENT_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \ PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \ BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE -from core.functionality import get_file_hash, create_event, create_job, \ - make_dir, write_yaml, write_notebook +from core.functionality import get_file_hash, create_watchdog_event, \ + create_job, make_dir, write_yaml, write_notebook from core.meow import create_rule from conductors import LocalPythonConductor from patterns import FileEventPattern @@ -69,17 +69,13 @@ class MeowTests(unittest.TestCase): job_dict = create_job( JOB_TYPE_PYTHON, - create_event( - EVENT_TYPE_WATCHDOG, + create_watchdog_event( file_path, rule, - { - WATCHDOG_BASE: TEST_MONITOR_BASE, - EVENT_RULE: rule, - WATCHDOG_HASH: file_hash - } + TEST_MONITOR_BASE, + file_hash ), - { + extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, PYTHON_FUNC:job_func, @@ -146,17 +142,13 @@ class MeowTests(unittest.TestCase): bad_job_dict = create_job( JOB_TYPE_PYTHON, - create_event( - EVENT_TYPE_WATCHDOG, + create_watchdog_event( file_path, rule, - { - WATCHDOG_BASE: TEST_MONITOR_BASE, - EVENT_RULE: rule, - WATCHDOG_HASH: file_hash - } + TEST_MONITOR_BASE, + file_hash ), - { + extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, PYTHON_FUNC:job_func, @@ -178,17 +170,13 @@ class MeowTests(unittest.TestCase): # Ensure execution can continue after one failed job good_job_dict = create_job( JOB_TYPE_PYTHON, - create_event( - EVENT_TYPE_WATCHDOG, + create_watchdog_event( file_path, rule, - { - WATCHDOG_BASE: TEST_MONITOR_BASE, - EVENT_RULE: rule, - WATCHDOG_HASH: file_hash - } + TEST_MONITOR_BASE, + file_hash ), - { + extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, PYTHON_FUNC:job_func, @@ -249,17 +237,13 @@ class MeowTests(unittest.TestCase): job_dict = create_job( JOB_TYPE_PYTHON, - create_event( - EVENT_TYPE_WATCHDOG, + create_watchdog_event( file_path, rule, - { - WATCHDOG_BASE: TEST_MONITOR_BASE, - EVENT_RULE: rule, - WATCHDOG_HASH: file_hash - } + TEST_MONITOR_BASE, + file_hash ), - { + extras={ JOB_PARAMETERS:{ "extra":"extra", "infile":file_path, diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 2b6ffee..39df83d 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -16,6 +16,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ from core.functionality import generate_id, wait, get_file_hash, rmtree, \ make_dir, parameterize_jupyter_notebook, create_event, create_job, \ replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \ + create_watchdog_event, create_fake_watchdog_event, \ KEYWORD_PATH, KEYWORD_REL_PATH, KEYWORD_DIR, KEYWORD_REL_DIR, \ KEYWORD_FILENAME, KEYWORD_PREFIX, KEYWORD_BASE, KEYWORD_EXTENSION, \ KEYWORD_JOB @@ -266,7 +267,7 @@ class CorrectnessTests(unittest.TestCase): self.assertEqual(event[EVENT_PATH], "path") self.assertEqual(event[EVENT_RULE], rule) - event2 = create_event("test2", "path2", rule, {"a":1}) + event2 = create_event("test2", "path2", rule, extras={"a":1}) self.assertEqual(type(event2), dict) self.assertTrue(EVENT_TYPE in event2.keys()) @@ -298,7 +299,7 @@ class CorrectnessTests(unittest.TestCase): EVENT_TYPE_WATCHDOG, "file_path", rule, - { + extras={ WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, WATCHDOG_HASH: "file_hash" @@ -308,7 +309,7 @@ class CorrectnessTests(unittest.TestCase): job_dict = create_job( JOB_TYPE_PYTHON, event, - { + extras={ JOB_PARAMETERS:{ "extra":"extra", "infile":"file_path", @@ -560,3 +561,101 @@ class CorrectnessTests(unittest.TestCase): self.assertFalse(os.path.exists(os.path.join(TEST_MONITOR_BASE, "A"))) self.assertFalse(os.path.exists( os.path.join(TEST_MONITOR_BASE, "A", "B"))) + + def testCreateWatchdogEvent(self)->None: + pattern = FileEventPattern( + "pattern", + "file_path", + "recipe_one", + "infile", + parameters={ + "extra":"A line from a test Pattern", + "outfile":"result_path" + }) + recipe = JupyterNotebookRecipe( + "recipe_one", APPENDING_NOTEBOOK) + + rule = create_rule(pattern, recipe) + + with self.assertRaises(TypeError): + event = create_watchdog_event("path", rule) + + event = create_watchdog_event("path", rule, "base", "hash") + + self.assertEqual(type(event), dict) + self.assertEqual(len(event.keys()), 5) + self.assertTrue(EVENT_TYPE in event.keys()) + self.assertTrue(EVENT_PATH in event.keys()) + self.assertTrue(EVENT_RULE in event.keys()) + self.assertTrue(WATCHDOG_BASE in event.keys()) + self.assertTrue(WATCHDOG_HASH in event.keys()) + self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) + self.assertEqual(event[EVENT_PATH], "path") + self.assertEqual(event[EVENT_RULE], rule) + self.assertEqual(event[WATCHDOG_BASE], "base") + self.assertEqual(event[WATCHDOG_HASH], "hash") + + event = create_watchdog_event( + "path2", rule, "base", "hash", extras={"a":1} + ) + + self.assertEqual(type(event), dict) + self.assertTrue(EVENT_TYPE in event.keys()) + self.assertTrue(EVENT_PATH in event.keys()) + self.assertTrue(EVENT_RULE in event.keys()) + self.assertTrue(WATCHDOG_BASE in event.keys()) + self.assertTrue(WATCHDOG_HASH in event.keys()) + self.assertEqual(len(event.keys()), 6) + self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) + self.assertEqual(event[EVENT_PATH], "path2") + self.assertEqual(event[EVENT_RULE], rule) + self.assertEqual(event["a"], 1) + self.assertEqual(event[WATCHDOG_BASE], "base") + self.assertEqual(event[WATCHDOG_HASH], "hash") + + def testCreateFakeWatchdogEvent(self)->None: + pattern = FileEventPattern( + "pattern", + "file_path", + "recipe_one", + "infile", + parameters={ + "extra":"A line from a test Pattern", + "outfile":"result_path" + }) + recipe = JupyterNotebookRecipe( + "recipe_one", APPENDING_NOTEBOOK) + + rule = create_rule(pattern, recipe) + + with self.assertRaises(TypeError): + event = create_fake_watchdog_event("path", rule) + + event = create_fake_watchdog_event("path", rule, "base") + + self.assertEqual(type(event), dict) + self.assertEqual(len(event.keys()), 4) + self.assertTrue(EVENT_TYPE in event.keys()) + self.assertTrue(EVENT_PATH in event.keys()) + self.assertTrue(EVENT_RULE in event.keys()) + self.assertTrue(WATCHDOG_BASE in event.keys()) + self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) + self.assertEqual(event[EVENT_PATH], "path") + self.assertEqual(event[EVENT_RULE], rule) + self.assertEqual(event[WATCHDOG_BASE], "base") + + event = create_fake_watchdog_event( + "path2", rule, "base", extras={"a":1} + ) + + self.assertEqual(type(event), dict) + self.assertTrue(EVENT_TYPE in event.keys()) + self.assertTrue(EVENT_PATH in event.keys()) + self.assertTrue(EVENT_RULE in event.keys()) + self.assertTrue(WATCHDOG_BASE in event.keys()) + self.assertEqual(len(event.keys()), 5) + self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) + self.assertEqual(event[EVENT_PATH], "path2") + self.assertEqual(event[EVENT_RULE], rule) + self.assertEqual(event["a"], 1) + self.assertEqual(event[WATCHDOG_BASE], "base") diff --git a/tests/test_meow.py b/tests/test_meow.py index 54a6663..7970344 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -1,7 +1,7 @@ import unittest -from typing import Any, Union +from typing import Any, Union, Tuple from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \ BaseHandler, BaseConductor, create_rules, create_rule @@ -198,7 +198,8 @@ class MeowTests(unittest.TestCase): pass def _is_valid_inputs(self, inputs:Any)->None: pass - def valid_handle_criteria(self, event:dict[str,Any])->bool: + def valid_handle_criteria(self, event:dict[str,Any] + )->Tuple[bool,str]: pass FullTestHandler() @@ -218,7 +219,8 @@ class MeowTests(unittest.TestCase): def execute(self, job:dict[str,Any])->None: pass - def valid_execute_criteria(self, job:dict[str,Any])->bool: + def valid_execute_criteria(self, job:dict[str,Any] + )->Tuple[bool,str]: pass FullTestConductor() diff --git a/tests/test_recipes.py b/tests/test_recipes.py index a2d4eb1..a973423 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -11,8 +11,8 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \ PARAMS_FILE, JOB_FILE, RESULT_FILE from core.correctness.validation import valid_job -from core.functionality import get_file_hash, create_job, create_event, \ - make_dir, write_yaml, write_notebook, read_yaml +from core.functionality import get_file_hash, create_job, \ + create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml from core.meow import create_rules, create_rule from patterns.file_event_pattern import FileEventPattern, SWEEP_START, \ SWEEP_STOP, SWEEP_JUMP @@ -351,17 +351,13 @@ class JupyterNotebookTests(unittest.TestCase): job_dict = create_job( JOB_TYPE_PYTHON, - create_event( - EVENT_TYPE_WATCHDOG, + create_watchdog_event( file_path, rule, - { - WATCHDOG_BASE: TEST_MONITOR_BASE, - EVENT_RULE: rule, - WATCHDOG_HASH: file_hash - } + TEST_MONITOR_BASE, + file_hash ), - { + extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, PYTHON_FUNC:job_func, diff --git a/tests/test_validation.py b/tests/test_validation.py index 5482ac1..8a0f169 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -9,10 +9,11 @@ from typing import Any, Union from core.correctness.validation import check_type, check_implementation, \ valid_string, valid_dict, valid_list, valid_existing_file_path, \ valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \ - setup_debugging + setup_debugging, valid_watchdog_event from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \ EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \ - JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE + JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \ + WATCHDOG_HASH from core.functionality import make_dir from shared import setup, teardown, TEST_MONITOR_BASE @@ -303,4 +304,36 @@ class CorrectnessTests(unittest.TestCase): with self.assertRaises(TypeError): setup_debugging(stream, "1") - #TODO test watchdog event dict + #Test watchdog event dict + def testWatchdogEventValidation(self)->None: + valid_watchdog_event({ + EVENT_TYPE: "test", + EVENT_PATH: "path", + EVENT_RULE: "rule", + WATCHDOG_HASH: "hash", + WATCHDOG_BASE: "base" + }) + + with self.assertRaises(KeyError): + valid_watchdog_event({ + EVENT_TYPE: "test", + EVENT_PATH: "path", + EVENT_RULE: "rule" + }) + + with self.assertRaises(KeyError): + valid_watchdog_event({ + EVENT_TYPE: "anything", + EVENT_PATH: "path", + EVENT_RULE: "rule", + "a": 1 + }) + + with self.assertRaises(KeyError): + valid_event({EVENT_TYPE: "test"}) + + with self.assertRaises(KeyError): + valid_event({"EVENT_TYPE": "test"}) + + with self.assertRaises(KeyError): + valid_event({}) \ No newline at end of file