diff --git a/core/base_handler.py b/core/base_handler.py index 31debef..6ce0ce2 100644 --- a/core/base_handler.py +++ b/core/base_handler.py @@ -14,9 +14,10 @@ from typing import Any, Tuple, Dict, Union from time import sleep from meow_base.core.vars import VALID_CHANNELS, EVENT_RULE, EVENT_PATH, \ - VALID_HANDLER_NAME_CHARS, META_FILE, JOB_ID, WATCHDOG_BASE, JOB_FILE, \ - JOB_PARAMETERS, get_drt_imp_msg + VALID_HANDLER_NAME_CHARS, META_FILE, JOB_ID, JOB_FILE, JOB_PARAMETERS, \ + get_drt_imp_msg from meow_base.core.meow import valid_event +from meow_base.patterns.file_event_pattern import WATCHDOG_HASH from meow_base.functionality.file_io import threadsafe_write_status, \ threadsafe_update_status, make_dir, write_file, lines_to_string from meow_base.functionality.validation import check_implementation, \ @@ -179,6 +180,7 @@ class BaseHandler: # Get updated job parameters # TODO replace this with generic implementation + from meow_base.patterns.file_event_pattern import WATCHDOG_BASE params_dict = replace_keywords( params_dict, meow_job[JOB_ID], @@ -242,8 +244,8 @@ class BaseHandler: "#!/bin/bash", "", "# Get job params", - "given_hash=$(grep 'file_hash: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)", - "event_path=$(grep 'event_path: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)", + f"given_hash=$(grep '{WATCHDOG_HASH}: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)", + f"event_path=$(grep '{EVENT_PATH}: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)", "", "echo event_path: $event_path", "echo given_hash: $given_hash", diff --git a/core/meow.py b/core/meow.py index 2cba545..39f1f14 100644 --- a/core/meow.py +++ b/core/meow.py @@ -12,21 +12,16 @@ from meow_base.core.rule import Rule from meow_base.functionality.validation import check_type from meow_base.core.vars import EVENT_TYPE, EVENT_PATH, \ JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \ - JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH + JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, EVENT_TIME # Required keys in event dict EVENT_KEYS = { EVENT_TYPE: str, EVENT_PATH: str, + EVENT_TIME: float, EVENT_RULE: Rule } -WATCHDOG_EVENT_KEYS = { - WATCHDOG_BASE: str, - WATCHDOG_HASH: str, - **EVENT_KEYS -} - # Required keys in job dict JOB_KEYS = { JOB_TYPE: str, @@ -59,6 +54,3 @@ def valid_event(event:Dict[str,Any])->None: def valid_job(job:Dict[str,Any])->None: """Check that a given dict expresses a meow job.""" valid_meow_dict(job, "Job", JOB_KEYS) - -def valid_watchdog_event(event:Dict[str,Any])->None: - valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS) diff --git a/core/vars.py b/core/vars.py index a8fdfa2..817257c 100644 --- a/core/vars.py +++ b/core/vars.py @@ -44,12 +44,8 @@ SHA256 = "sha256" # meow events EVENT_TYPE = "event_type" EVENT_PATH = "event_path" -EVENT_RULE = "rule" - -# watchdog events -EVENT_TYPE_WATCHDOG = "watchdog" -WATCHDOG_BASE = "monitor_base" -WATCHDOG_HASH = "file_hash" +EVENT_RULE = "event_rule" +EVENT_TIME = "event_time" # inotify events FILE_CREATE_EVENT = "file_created" diff --git a/functionality/meow.py b/functionality/meow.py index 035d73c..ce70f0f 100644 --- a/functionality/meow.py +++ b/functionality/meow.py @@ -13,11 +13,10 @@ from meow_base.core.base_recipe import BaseRecipe from meow_base.core.rule import Rule from meow_base.functionality.validation import check_type, valid_dict, \ valid_list -from meow_base.core.vars import EVENT_PATH, EVENT_RULE, \ - EVENT_TYPE, EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \ +from meow_base.core.vars import EVENT_PATH, EVENT_RULE, EVENT_TIME, \ + EVENT_TYPE, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \ JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \ - JOB_TYPE, STATUS_CREATING, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \ - SWEEP_START, SWEEP_STOP + JOB_TYPE, STATUS_CREATING, SWEEP_JUMP, SWEEP_START, SWEEP_STOP from meow_base.functionality.naming import generate_job_id # mig trigger keyword replacements @@ -104,32 +103,17 @@ def create_parameter_sweep(variable_name:str, start:Union[int,float,complex], } } -def create_event(event_type:str, path:str, rule:Any, extras:Dict[Any,Any]={} - )->Dict[Any,Any]: +def create_event(event_type:str, path:str, rule:Any, time:float, + extras:Dict[Any,Any]={})->Dict[Any,Any]: """Function to create a MEOW dictionary.""" return { **extras, EVENT_PATH: path, EVENT_TYPE: event_type, - EVENT_RULE: rule + EVENT_RULE: rule, + EVENT_TIME: time } -def create_watchdog_event(path:str, rule:Any, base:str, hash:str, - extras:Dict[Any,Any]={})->Dict[Any,Any]: - """Function to create a MEOW event dictionary.""" - return create_event( - EVENT_TYPE_WATCHDOG, - path, - rule, - extras={ - **extras, - **{ - WATCHDOG_HASH: hash, - WATCHDOG_BASE: base - } - } - ) - def create_job_metadata_dict(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={})->Dict[Any,Any]: """Function to create a MEOW job dictionary.""" diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 73326ad..07c719f 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -20,6 +20,7 @@ from watchdog.events import PatternMatchingEventHandler from meow_base.core.base_recipe import BaseRecipe from meow_base.core.base_monitor import BaseMonitor from meow_base.core.base_pattern import BasePattern +from meow_base.core.meow import EVENT_KEYS, valid_meow_dict from meow_base.core.rule import Rule from meow_base.functionality.validation import check_type, valid_string, \ valid_dict, valid_list, valid_dir_path @@ -30,7 +31,7 @@ from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \ DIR_RETROACTIVE_EVENT from meow_base.functionality.debug import setup_debugging, print_debug from meow_base.functionality.hashing import get_hash -from meow_base.functionality.meow import create_watchdog_event +from meow_base.functionality.meow import create_event # Events that are monitored by default _DEFAULT_MASK = [ @@ -41,6 +42,38 @@ _DEFAULT_MASK = [ FILE_CLOSED_EVENT ] +# watchdog events +EVENT_TYPE_WATCHDOG = "watchdog" +WATCHDOG_BASE = "monitor_base" +WATCHDOG_HASH = "file_hash" + +WATCHDOG_EVENT_KEYS = { + WATCHDOG_BASE: str, + WATCHDOG_HASH: str, + **EVENT_KEYS +} + +def create_watchdog_event(path:str, rule:Any, base:str, time:float, + hash:str, extras:Dict[Any,Any]={})->Dict[Any,Any]: + """Function to create a MEOW event dictionary.""" + return create_event( + EVENT_TYPE_WATCHDOG, + path, + rule, + time, + extras={ + **extras, + **{ + WATCHDOG_HASH: hash, + WATCHDOG_BASE: base + } + } + ) + +def valid_watchdog_event(event:Dict[str,Any])->None: + valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS) + + class FileEventPattern(BasePattern): # The path at which events will trigger this pattern triggering_path:str @@ -239,6 +272,7 @@ class WatchdogMonitor(BaseMonitor): event.src_path, rule, self.base_dir, + event.time_stamp, get_hash(event.src_path, SHA256) ) print_debug(self._print_target, self.debug_level, @@ -300,6 +334,7 @@ class WatchdogMonitor(BaseMonitor): globble, rule, self.base_dir, + time(), get_hash(globble, SHA256) ) print_debug(self._print_target, self.debug_level, diff --git a/recipes/bash_recipe.py b/recipes/bash_recipe.py index 715f0fd..6d3ec81 100644 --- a/recipes/bash_recipe.py +++ b/recipes/bash_recipe.py @@ -11,13 +11,13 @@ from meow_base.core.meow import valid_event from meow_base.functionality.validation import check_type, valid_dict, \ valid_string, valid_dir_path from meow_base.core.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \ - VALID_VARIABLE_NAME_CHARS, EVENT_RULE, EVENT_TYPE, EVENT_TYPE_WATCHDOG, \ + VALID_VARIABLE_NAME_CHARS, EVENT_RULE, EVENT_TYPE, \ JOB_TYPE_BASH from meow_base.functionality.debug import setup_debugging, print_debug from meow_base.functionality.file_io import valid_path, make_dir, write_file, \ lines_to_string from meow_base.functionality.parameterisation import parameterize_bash_script - +from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG class BashRecipe(BaseRecipe): # A path to the bash script used to create this recipe diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index 42be6ae..6405735 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -18,13 +18,14 @@ from meow_base.core.meow import valid_event from meow_base.functionality.validation import check_type, valid_string, \ valid_dict, valid_path, valid_dir_path, valid_existing_file_path from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \ - DEBUG_INFO, EVENT_TYPE_WATCHDOG, DEFAULT_JOB_QUEUE_DIR, \ + DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \ JOB_TYPE_PAPERMILL, EVENT_RULE, EVENT_TYPE, EVENT_RULE from meow_base.functionality.debug import setup_debugging, print_debug from meow_base.functionality.file_io import make_dir, read_notebook, \ write_notebook from meow_base.functionality.parameterisation import \ parameterize_jupyter_notebook +from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG class JupyterNotebookRecipe(BaseRecipe): # A path to the jupyter notebook used to create this recipe diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index b31ecc8..4dd6a44 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -17,13 +17,13 @@ from meow_base.core.meow import valid_event from meow_base.functionality.validation import check_script, valid_string, \ valid_dict, valid_dir_path from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \ - DEBUG_INFO, EVENT_TYPE_WATCHDOG, DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, \ + DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, \ JOB_TYPE_PYTHON, EVENT_TYPE, EVENT_RULE from meow_base.functionality.debug import setup_debugging, print_debug from meow_base.functionality.file_io import make_dir, write_file, \ lines_to_string from meow_base.functionality.parameterisation import parameterize_python_script - +from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG class PythonRecipe(BaseRecipe): def __init__(self, name:str, recipe:List[str], parameters:Dict[str,Any]={}, diff --git a/tests/test_conductors.py b/tests/test_conductors.py index b956f0e..8623824 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -5,6 +5,7 @@ import unittest from datetime import datetime from multiprocessing import Pipe +from time import time from typing import Dict from meow_base.core.vars import JOB_TYPE_PYTHON, SHA256, \ @@ -12,15 +13,16 @@ from meow_base.core.vars import JOB_TYPE_PYTHON, SHA256, \ JOB_EVENT, META_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \ JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \ JOB_CREATE_TIME, JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ - EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_FILE + JOB_TYPE_BASH, JOB_FILE from meow_base.conductors import LocalPythonConductor, LocalBashConductor from meow_base.functionality.file_io import read_file, read_yaml, write_file, \ write_yaml, lines_to_string, make_dir, threadsafe_read_status from meow_base.functionality.hashing import get_hash -from meow_base.functionality.meow import create_watchdog_event, create_job_metadata_dict, \ +from meow_base.functionality.meow import create_job_metadata_dict, \ create_rule from meow_base.functionality.parameterisation import parameterize_bash_script -from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.patterns.file_event_pattern import FileEventPattern, \ + EVENT_TYPE_WATCHDOG, create_watchdog_event from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler @@ -101,6 +103,7 @@ class PythonTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ) @@ -192,6 +195,7 @@ class PythonTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ) @@ -268,6 +272,7 @@ class PythonTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ), extras={ @@ -333,6 +338,7 @@ class PythonTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ), extras={ @@ -401,6 +407,7 @@ class PythonTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ), extras={ @@ -598,6 +605,7 @@ class BashTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ) @@ -689,6 +697,7 @@ class BashTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ), extras={ @@ -755,6 +764,7 @@ class BashTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ), extras={ @@ -839,6 +849,7 @@ class BashTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ), extras={ @@ -908,6 +919,7 @@ class BashTests(unittest.TestCase): file_path, rule, TEST_MONITOR_BASE, + time(), file_hash ), extras={ diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 2f9450d..daf164d 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -8,16 +8,16 @@ from datetime import datetime from multiprocessing import Pipe, Queue from os.path import basename from sys import prefix, base_prefix -from time import sleep +from time import sleep, time from typing import Dict +from meow_base.core.meow import EVENT_KEYS from meow_base.core.rule import Rule from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ - SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, LOCK_EXT, \ - WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, \ + SHA256, EVENT_TYPE, EVENT_PATH, LOCK_EXT, EVENT_RULE, JOB_PARAMETERS, \ PYTHON_FUNC, JOB_ID, JOB_EVENT, JOB_ERROR, STATUS_DONE, \ JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ - JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL, STATUS_CREATING + JOB_REQUIREMENTS, JOB_TYPE_PAPERMILL, STATUS_CREATING from meow_base.functionality.debug import setup_debugging from meow_base.functionality.file_io import lines_to_string, make_dir, \ read_file, read_file_lines, read_notebook, read_yaml, rmtree, write_file, \ @@ -28,7 +28,7 @@ from meow_base.functionality.meow import KEYWORD_BASE, KEYWORD_DIR, \ KEYWORD_EXTENSION, KEYWORD_FILENAME, KEYWORD_JOB, KEYWORD_PATH, \ KEYWORD_PREFIX, KEYWORD_REL_DIR, KEYWORD_REL_PATH, \ create_event, create_job_metadata_dict, create_rule, create_rules, \ - create_watchdog_event, replace_keywords, create_parameter_sweep + replace_keywords, create_parameter_sweep from meow_base.functionality.naming import _generate_id from meow_base.functionality.parameterisation import \ parameterize_jupyter_notebook, parameterize_python_script @@ -36,7 +36,8 @@ from meow_base.functionality.process_io import wait from meow_base.functionality.requirements import REQUIREMENT_PYTHON, \ REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, REQ_PYTHON_VERSION, \ create_python_requirements, check_requirements -from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.patterns.file_event_pattern import FileEventPattern, \ + EVENT_TYPE_WATCHDOG, WATCHDOG_BASE, WATCHDOG_HASH from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe from shared import TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, \ COMPLETE_PYTHON_SCRIPT, valid_recipe_two, valid_recipe_one, \ @@ -622,24 +623,26 @@ class MeowTests(unittest.TestCase): rule = create_rule(pattern, recipe) - event = create_event("test", "path", rule) + event = create_event("test", "path", rule, time()) self.assertEqual(type(event), dict) - self.assertEqual(len(event.keys()), 3) - self.assertTrue(EVENT_TYPE in event.keys()) - self.assertTrue(EVENT_PATH in event.keys()) - self.assertTrue(EVENT_RULE in event.keys()) + self.assertEqual(len(event.keys()), len(EVENT_KEYS)) + for key, value in EVENT_KEYS.items(): + self.assertTrue(key in event.keys()) + self.assertIsInstance(event[key], value) self.assertEqual(event[EVENT_TYPE], "test") self.assertEqual(event[EVENT_PATH], "path") self.assertEqual(event[EVENT_RULE], rule) - event2 = create_event("test2", "path2", rule, extras={"a":1}) + event2 = create_event( + "test2", "path2", rule, time(), extras={"a":1} + ) self.assertEqual(type(event2), dict) - self.assertTrue(EVENT_TYPE in event2.keys()) - self.assertTrue(EVENT_PATH in event.keys()) - self.assertTrue(EVENT_RULE in event.keys()) - self.assertEqual(len(event2.keys()), 4) + self.assertEqual(len(event.keys()), len(EVENT_KEYS)) + for key, value in EVENT_KEYS.items(): + self.assertTrue(key in event.keys()) + self.assertIsInstance(event[key], value) self.assertEqual(event2[EVENT_TYPE], "test2") self.assertEqual(event2[EVENT_PATH], "path2") self.assertEqual(event2[EVENT_RULE], rule) @@ -665,6 +668,7 @@ class MeowTests(unittest.TestCase): EVENT_TYPE_WATCHDOG, "file_path", rule, + time(), extras={ WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, @@ -705,58 +709,6 @@ class MeowTests(unittest.TestCase): self.assertIn(JOB_REQUIREMENTS, job_dict) self.assertEqual(job_dict[JOB_REQUIREMENTS], {}) - # Test creation of watchdog event dict - def testCreateWatchdogEvent(self)->None: - pattern = FileEventPattern( - "pattern", - "file_path", - "recipe_one", - "infile", - parameters={ - "extra":"A line from a test Pattern", - "outfile":"result_path" - }) - recipe = JupyterNotebookRecipe( - "recipe_one", APPENDING_NOTEBOOK) - - rule = create_rule(pattern, recipe) - - with self.assertRaises(TypeError): - event = create_watchdog_event("path", rule) - - event = create_watchdog_event("path", rule, "base", "hash") - - self.assertEqual(type(event), dict) - self.assertEqual(len(event.keys()), 5) - self.assertTrue(EVENT_TYPE in event.keys()) - self.assertTrue(EVENT_PATH in event.keys()) - self.assertTrue(EVENT_RULE in event.keys()) - self.assertTrue(WATCHDOG_BASE in event.keys()) - self.assertTrue(WATCHDOG_HASH in event.keys()) - self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) - self.assertEqual(event[EVENT_PATH], "path") - self.assertEqual(event[EVENT_RULE], rule) - self.assertEqual(event[WATCHDOG_BASE], "base") - self.assertEqual(event[WATCHDOG_HASH], "hash") - - event = create_watchdog_event( - "path2", rule, "base", "hash", extras={"a":1} - ) - - self.assertEqual(type(event), dict) - self.assertTrue(EVENT_TYPE in event.keys()) - self.assertTrue(EVENT_PATH in event.keys()) - self.assertTrue(EVENT_RULE in event.keys()) - self.assertTrue(WATCHDOG_BASE in event.keys()) - self.assertTrue(WATCHDOG_HASH in event.keys()) - self.assertEqual(len(event.keys()), 6) - self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) - self.assertEqual(event[EVENT_PATH], "path2") - self.assertEqual(event[EVENT_RULE], rule) - self.assertEqual(event["a"], 1) - self.assertEqual(event[WATCHDOG_BASE], "base") - self.assertEqual(event[WATCHDOG_HASH], "hash") - # Test that replace_keywords replaces MEOW keywords in a given dictionary def testReplaceKeywords(self)->None: test_dict = { diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 4044838..1902323 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -3,19 +3,22 @@ import io import os import unittest +from datetime import datetime from multiprocessing import Pipe -from time import sleep +from time import sleep, time from meow_base.core.vars import FILE_CREATE_EVENT, EVENT_TYPE, \ - EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \ + EVENT_RULE, EVENT_PATH, SWEEP_START, \ SWEEP_JUMP, SWEEP_STOP, DIR_EVENTS from meow_base.functionality.file_io import make_dir +from meow_base.functionality.meow import create_rule from meow_base.patterns.file_event_pattern import FileEventPattern, \ - WatchdogMonitor, _DEFAULT_MASK + WatchdogMonitor, _DEFAULT_MASK, WATCHDOG_HASH, WATCHDOG_BASE, \ + EVENT_TYPE_WATCHDOG, WATCHDOG_EVENT_KEYS, create_watchdog_event from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe from meow_base.recipes.python_recipe import PythonRecipe from shared import BAREBONES_NOTEBOOK, TEST_MONITOR_BASE, \ - COUNTING_PYTHON_SCRIPT, setup, teardown + COUNTING_PYTHON_SCRIPT, APPENDING_NOTEBOOK, setup, teardown def patterns_equal(tester, pattern_one, pattern_two): @@ -196,6 +199,67 @@ class WatchdogMonitorTests(unittest.TestCase): super().tearDown() teardown() + # Test creation of watchdog event dict + def testCreateWatchdogEvent(self)->None: + pattern = FileEventPattern( + "pattern", + "file_path", + "recipe_one", + "infile", + parameters={ + "extra":"A line from a test Pattern", + "outfile":"result_path" + }) + recipe = JupyterNotebookRecipe( + "recipe_one", APPENDING_NOTEBOOK) + + rule = create_rule(pattern, recipe) + + with self.assertRaises(TypeError): + event = create_watchdog_event("path", rule) + + event = create_watchdog_event( + "path", rule, "base", time(), "hash") + + self.assertEqual(type(event), dict) + self.assertEqual(len(event.keys()), len(WATCHDOG_EVENT_KEYS)) + for key, value in WATCHDOG_EVENT_KEYS.items(): + self.assertTrue(key in event.keys()) + self.assertIsInstance(event[key], value) + self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) + self.assertEqual(event[EVENT_PATH], "path") + self.assertEqual(event[EVENT_RULE], rule) + self.assertEqual(event[WATCHDOG_BASE], "base") + self.assertEqual(event[WATCHDOG_HASH], "hash") + + event = create_watchdog_event( + "path2", + rule, + "base", + time(), + "hash", + extras={"a":1} + ) + + self.assertEqual(type(event), dict) + self.assertTrue(EVENT_TYPE in event.keys()) + self.assertTrue(EVENT_PATH in event.keys()) + self.assertTrue(EVENT_RULE in event.keys()) + self.assertTrue(WATCHDOG_BASE in event.keys()) + self.assertTrue(WATCHDOG_HASH in event.keys()) + self.assertEqual(len(event.keys()), len(WATCHDOG_EVENT_KEYS)+1) + for key, value in WATCHDOG_EVENT_KEYS.items(): + self.assertTrue(key in event.keys()) + self.assertIsInstance(event[key], value) + self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG) + self.assertEqual(event[EVENT_PATH], "path2") + self.assertEqual(event[EVENT_RULE], rule) + self.assertEqual(event["a"], 1) + self.assertEqual(event[WATCHDOG_BASE], "base") + self.assertEqual(event[WATCHDOG_HASH], "hash") + + #TODO test valid watchdog event + # Test WatchdogMonitor created def testWatchdogMonitorMinimum(self)->None: from_monitor = Pipe() diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 680942c..50b53c8 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -4,18 +4,19 @@ import os import unittest from multiprocessing import Pipe +from time import time from meow_base.core.meow import valid_job -from meow_base.core.vars import EVENT_TYPE, WATCHDOG_BASE, \ - EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \ +from meow_base.core.vars import EVENT_TYPE, EVENT_RULE, EVENT_PATH, SHA256, \ JOB_PARAMETERS, JOB_FILE, META_FILE, SWEEP_STOP, SWEEP_JUMP, \ - SWEEP_START + SWEEP_START, EVENT_TIME from meow_base.core.rule import Rule from meow_base.functionality.file_io import read_yaml, write_notebook, \ threadsafe_read_status from meow_base.functionality.hashing import get_hash from meow_base.functionality.meow import create_rules, create_rule -from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.patterns.file_event_pattern import FileEventPattern, \ + WATCHDOG_BASE, WATCHDOG_HASH, EVENT_TYPE_WATCHDOG from meow_base.recipes.bash_recipe import BashRecipe, BashHandler from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler, get_recipe_from_notebook @@ -163,6 +164,7 @@ class PapermillHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -214,6 +216,7 @@ class PapermillHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -284,6 +287,7 @@ class PapermillHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -348,21 +352,24 @@ class PapermillHandlerTests(unittest.TestCase): status, _ = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: "type", - EVENT_RULE: rule + EVENT_RULE: rule.name, + EVENT_TIME: time() }) self.assertFalse(status) status, _ = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: EVENT_TYPE_WATCHDOG, - EVENT_RULE: "rule" + EVENT_RULE: "rule", + EVENT_TIME: time() }) self.assertFalse(status) status, _ = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: EVENT_TYPE_WATCHDOG, - EVENT_RULE: rule + EVENT_RULE: rule, + EVENT_TIME: time() }) self.assertTrue(status) @@ -432,6 +439,7 @@ class PapermillHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -577,6 +585,7 @@ class PythonHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -628,6 +637,7 @@ class PythonHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -698,6 +708,7 @@ class PythonHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -763,21 +774,24 @@ class PythonHandlerTests(unittest.TestCase): status, _ = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: "type", - EVENT_RULE: rule + EVENT_RULE: rule, + EVENT_TIME: time() }) self.assertFalse(status) status, _ = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: EVENT_TYPE_WATCHDOG, - EVENT_RULE: "rule" + EVENT_RULE: "rule", + EVENT_TIME: time() }) self.assertFalse(status) status, s = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: EVENT_TYPE_WATCHDOG, - EVENT_RULE: rule + EVENT_RULE: rule, + EVENT_TIME: time() }) self.assertTrue(status) @@ -834,6 +848,7 @@ class PythonHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -979,6 +994,7 @@ class BashHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -1030,6 +1046,7 @@ class BashHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -1100,6 +1117,7 @@ class BashHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -1176,6 +1194,7 @@ class BashHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) @@ -1221,21 +1240,24 @@ class BashHandlerTests(unittest.TestCase): status, _ = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: "type", - EVENT_RULE: rule + EVENT_RULE: rule, + EVENT_TIME: time() }) self.assertFalse(status) status, _ = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: EVENT_TYPE_WATCHDOG, - EVENT_RULE: "rule" + EVENT_RULE: "rule", + EVENT_TIME: time() }) self.assertFalse(status) status, s = ph.valid_handle_criteria({ EVENT_PATH: "path", EVENT_TYPE: EVENT_TYPE_WATCHDOG, - EVENT_RULE: rule + EVENT_RULE: rule, + EVENT_TIME: time() }) self.assertTrue(status) @@ -1292,6 +1314,7 @@ class BashHandlerTests(unittest.TestCase): EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), WATCHDOG_BASE: TEST_MONITOR_BASE, EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: get_hash( os.path.join(TEST_MONITOR_BASE, "A"), SHA256 ) diff --git a/tests/test_validation.py b/tests/test_validation.py index 66a3528..c68a2ed 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -3,20 +3,21 @@ import unittest import os from datetime import datetime +from time import time from typing import Any, Union -from meow_base.core.meow import valid_event, valid_job, \ - valid_watchdog_event +from meow_base.core.meow import valid_event, valid_job from meow_base.functionality.validation import check_type, \ check_implementation, valid_string, valid_dict, valid_list, \ valid_existing_file_path, valid_dir_path, valid_non_existing_path, \ check_callable, valid_natural, valid_dict_multiple_types from meow_base.core.vars import VALID_NAME_CHARS, SHA256, \ EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \ - JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \ - WATCHDOG_BASE, WATCHDOG_HASH + JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, EVENT_TIME from meow_base.functionality.file_io import make_dir from meow_base.functionality.meow import create_rule +from meow_base.patterns.file_event_pattern import WATCHDOG_BASE, \ + WATCHDOG_HASH, valid_watchdog_event from shared import TEST_MONITOR_BASE, valid_pattern_one, valid_recipe_one, \ setup, teardown @@ -313,12 +314,14 @@ class MeowTests(unittest.TestCase): valid_event({ EVENT_TYPE: "test", EVENT_PATH: "path", - EVENT_RULE: rule + EVENT_RULE: rule, + EVENT_TIME: time() }) valid_event({ EVENT_TYPE: "anything", EVENT_PATH: "path", EVENT_RULE: rule, + EVENT_TIME: time(), "a": 1 }) @@ -361,6 +364,7 @@ class MeowTests(unittest.TestCase): EVENT_TYPE: "test", EVENT_PATH: "path", EVENT_RULE: rule, + EVENT_TIME: time(), WATCHDOG_HASH: "hash", WATCHDOG_BASE: "base" }) @@ -369,7 +373,8 @@ class MeowTests(unittest.TestCase): valid_watchdog_event({ EVENT_TYPE: "test", EVENT_PATH: "path", - EVENT_RULE: "rule" + EVENT_RULE: "rule", + EVENT_TIME: time() }) with self.assertRaises(KeyError): @@ -377,6 +382,7 @@ class MeowTests(unittest.TestCase): EVENT_TYPE: "anything", EVENT_PATH: "path", EVENT_RULE: "rule", + EVENT_TIME: time(), "a": 1 })