standardised event construction and validation a bit more
This commit is contained in:
@ -9,7 +9,7 @@ import os
|
||||
import shutil
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from typing import Any, Tuple
|
||||
|
||||
from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
|
||||
STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
|
||||
@ -25,16 +25,16 @@ class LocalPythonConductor(BaseConductor):
|
||||
def __init__(self)->None:
|
||||
super().__init__()
|
||||
|
||||
def valid_execute_criteria(self, job:dict[str,Any])->bool:
|
||||
def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]:
|
||||
"""Function to determine given an job defintion, if this conductor can
|
||||
process it or not. This conductor will accept any Python job type"""
|
||||
try:
|
||||
valid_job(job)
|
||||
if job[JOB_TYPE] == JOB_TYPE_PYTHON:
|
||||
return True
|
||||
except:
|
||||
return True, ""
|
||||
except Exception as e:
|
||||
pass
|
||||
return False
|
||||
return False, str(e)
|
||||
|
||||
def execute(self, job:dict[str,Any])->None:
|
||||
valid_job(job)
|
||||
|
@ -12,7 +12,8 @@ from typing import Any, _SpecialForm, Union, Tuple, get_origin, get_args
|
||||
|
||||
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \
|
||||
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
|
||||
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE
|
||||
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
|
||||
WATCHDOG_BASE
|
||||
|
||||
# Required keys in event dict
|
||||
EVENT_KEYS = {
|
||||
@ -23,6 +24,11 @@ EVENT_KEYS = {
|
||||
EVENT_RULE: Any
|
||||
}
|
||||
|
||||
WATCHDOG_EVENT_KEYS = {
|
||||
WATCHDOG_BASE: str,
|
||||
**EVENT_KEYS
|
||||
}
|
||||
|
||||
# Required keys in job dict
|
||||
JOB_KEYS = {
|
||||
JOB_TYPE: str,
|
||||
@ -254,3 +260,6 @@ def valid_event(event:dict[str,Any])->None:
|
||||
def valid_job(job:dict[str,Any])->None:
|
||||
"""Check that a given dict expresses a meow job."""
|
||||
valid_meow_dict(job, "Job", JOB_KEYS)
|
||||
|
||||
def valid_watchdog_event(event:dict[str,Any])->None:
|
||||
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)
|
||||
|
@ -20,7 +20,8 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
||||
VALID_CHANNELS, HASH_BUFFER_SIZE, SHA256, DEBUG_WARNING, DEBUG_INFO, \
|
||||
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
|
||||
JOB_RECIPE, JOB_RULE, EVENT_RULE, JOB_STATUS, STATUS_QUEUED, \
|
||||
JOB_CREATE_TIME, JOB_REQUIREMENTS
|
||||
JOB_CREATE_TIME, JOB_REQUIREMENTS, WATCHDOG_BASE, WATCHDOG_HASH, \
|
||||
EVENT_TYPE_WATCHDOG, JOB_TYPE_PYTHON
|
||||
|
||||
# mig trigger keyword replacements
|
||||
KEYWORD_PATH = "{PATH}"
|
||||
@ -283,16 +284,45 @@ def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
|
||||
|
||||
return new_dict
|
||||
|
||||
def create_event(event_type:str, path:str, rule:Any, source:dict[Any,Any]={}
|
||||
def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={}
|
||||
)->dict[Any,Any]:
|
||||
return {
|
||||
**source,
|
||||
**extras,
|
||||
EVENT_PATH: path,
|
||||
EVENT_TYPE: event_type,
|
||||
EVENT_RULE: rule
|
||||
}
|
||||
|
||||
def create_job(job_type:str, event:dict[str,Any], source:dict[Any,Any]={}
|
||||
def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
|
||||
extras:dict[Any,Any]={})->dict[Any,Any]:
|
||||
return create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
path,
|
||||
rule,
|
||||
extras={
|
||||
**extras,
|
||||
**{
|
||||
WATCHDOG_HASH: hash,
|
||||
WATCHDOG_BASE: base
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def create_fake_watchdog_event(path:str, rule:Any, base:str,
|
||||
extras:dict[Any,Any]={})->dict[Any,Any]:
|
||||
return create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
path,
|
||||
rule,
|
||||
extras={
|
||||
**extras,
|
||||
**{
|
||||
WATCHDOG_BASE: base
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
|
||||
)->dict[Any,Any]:
|
||||
job_dict = {
|
||||
#TODO compress event?
|
||||
@ -307,4 +337,4 @@ def create_job(job_type:str, event:dict[str,Any], source:dict[Any,Any]={}
|
||||
JOB_REQUIREMENTS: event[EVENT_RULE].recipe.requirements
|
||||
}
|
||||
|
||||
return {**source, **job_dict}
|
||||
return {**extras, **job_dict}
|
||||
|
@ -11,7 +11,7 @@ import inspect
|
||||
import sys
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any, Union
|
||||
from typing import Any, Union, Tuple
|
||||
|
||||
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
|
||||
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, VALID_CHANNELS, \
|
||||
@ -323,8 +323,7 @@ class BaseHandler:
|
||||
raise TypeError(msg)
|
||||
return object.__new__(cls)
|
||||
|
||||
# TODO also implement something like me from conductor
|
||||
def valid_handle_criteria(self, event:dict[str,Any])->bool:
|
||||
def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]:
|
||||
"""Function to determine given an event defintion, if this handler can
|
||||
process it or not. Must be implemented by any child process."""
|
||||
pass
|
||||
@ -350,7 +349,7 @@ class BaseConductor:
|
||||
raise TypeError(msg)
|
||||
return object.__new__(cls)
|
||||
|
||||
def valid_execute_criteria(self, job:dict[str,Any])->bool:
|
||||
def valid_execute_criteria(self, job:dict[str,Any])->Tuple[bool,str]:
|
||||
"""Function to determine given an job defintion, if this conductor can
|
||||
process it or not. Must be implemented by any child process."""
|
||||
pass
|
||||
|
@ -10,13 +10,12 @@ import os
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from inspect import signature
|
||||
from multiprocessing import Pipe
|
||||
from random import randrange
|
||||
from typing import Any, Union
|
||||
|
||||
from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
|
||||
VALID_CHANNELS, JOB_TYPE, JOB_ID, META_FILE
|
||||
VALID_CHANNELS, JOB_ID, META_FILE
|
||||
from core.correctness.validation import setup_debugging, check_type, \
|
||||
valid_list
|
||||
from core.functionality import print_debug, wait, read_yaml
|
||||
@ -104,7 +103,7 @@ class MeowRunner:
|
||||
valid_handlers = []
|
||||
for handler in self.handlers:
|
||||
try:
|
||||
valid = handler.valid_handle_criteria(event)
|
||||
valid, _ = handler.valid_handle_criteria(event)
|
||||
if valid:
|
||||
valid_handlers.append(handler)
|
||||
except Exception as e:
|
||||
@ -155,7 +154,8 @@ class MeowRunner:
|
||||
valid_conductors = []
|
||||
for conductor in self.conductors:
|
||||
try:
|
||||
valid = conductor.valid_execute_criteria(job)
|
||||
valid, _ = \
|
||||
conductor.valid_execute_criteria(job)
|
||||
if valid:
|
||||
valid_conductors.append(conductor)
|
||||
except Exception as e:
|
||||
|
@ -25,7 +25,8 @@ from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
|
||||
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
|
||||
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
|
||||
WATCHDOG_BASE, FILE_RETROACTIVE_EVENT, WATCHDOG_HASH, SHA256
|
||||
from core.functionality import print_debug, create_event, get_file_hash
|
||||
from core.functionality import print_debug, create_watchdog_event, \
|
||||
get_file_hash, create_fake_watchdog_event
|
||||
from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \
|
||||
create_rule
|
||||
|
||||
@ -236,17 +237,11 @@ class WatchdogMonitor(BaseMonitor):
|
||||
|
||||
# If matched, the create a watchdog event
|
||||
if direct_hit or recursive_hit:
|
||||
meow_event = create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
meow_event = create_watchdog_event(
|
||||
event.src_path,
|
||||
rule,
|
||||
{
|
||||
WATCHDOG_BASE: self.base_dir,
|
||||
WATCHDOG_HASH: get_file_hash(
|
||||
event.src_path,
|
||||
SHA256
|
||||
)
|
||||
}
|
||||
self.base_dir,
|
||||
get_file_hash(event.src_path, SHA256)
|
||||
)
|
||||
print_debug(self._print_target, self.debug_level,
|
||||
f"Event at {src_path} of type {event_type} hit rule "
|
||||
@ -534,11 +529,10 @@ class WatchdogMonitor(BaseMonitor):
|
||||
# For each file create a fake event.
|
||||
for globble in globbed:
|
||||
|
||||
meow_event = create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
meow_event = create_fake_watchdog_event(
|
||||
globble,
|
||||
rule,
|
||||
{ WATCHDOG_BASE: self.base_dir }
|
||||
self.base_dir
|
||||
)
|
||||
print_debug(self._print_target, self.debug_level,
|
||||
f"Retroactive event for file at at {globble} hit rule "
|
||||
|
@ -10,7 +10,7 @@ import itertools
|
||||
import nbformat
|
||||
import sys
|
||||
|
||||
from typing import Any
|
||||
from typing import Any, Tuple
|
||||
|
||||
from core.correctness.validation import check_type, valid_string, \
|
||||
valid_dict, valid_path, valid_existing_dir_path, setup_debugging, \
|
||||
@ -123,7 +123,7 @@ class PapermillHandler(BaseHandler):
|
||||
yaml_dict[value[0]] = value[1]
|
||||
self.setup_job(event, yaml_dict)
|
||||
|
||||
def valid_handle_criteria(self, event:dict[str,Any])->bool:
|
||||
def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]:
|
||||
"""Function to determine given an event defintion, if this handler can
|
||||
process it or not. This handler accepts events from watchdog with
|
||||
jupyter notebook recipes."""
|
||||
@ -131,10 +131,10 @@ class PapermillHandler(BaseHandler):
|
||||
valid_event(event)
|
||||
if type(event[EVENT_RULE].recipe) == JupyterNotebookRecipe \
|
||||
and event[EVENT_TYPE] == EVENT_TYPE_WATCHDOG:
|
||||
return True
|
||||
except:
|
||||
return True, ""
|
||||
except Exception as e:
|
||||
pass
|
||||
return False
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def _is_valid_handler_base(self, handler_base)->None:
|
||||
@ -153,7 +153,7 @@ class PapermillHandler(BaseHandler):
|
||||
meow_job = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
event,
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:yaml_dict,
|
||||
JOB_HASH: event[WATCHDOG_HASH],
|
||||
PYTHON_FUNC:job_func,
|
||||
|
@ -10,7 +10,7 @@ import itertools
|
||||
import nbformat
|
||||
import sys
|
||||
|
||||
from typing import Any
|
||||
from typing import Any, Tuple
|
||||
|
||||
from core.correctness.validation import check_type, valid_string, \
|
||||
valid_dict, valid_event, valid_existing_dir_path, setup_debugging
|
||||
@ -114,7 +114,7 @@ class PythonHandler(BaseHandler):
|
||||
yaml_dict[value[0]] = value[1]
|
||||
self.setup_job(event, yaml_dict)
|
||||
|
||||
def valid_handle_criteria(self, event:dict[str,Any])->bool:
|
||||
def valid_handle_criteria(self, event:dict[str,Any])->Tuple[bool,str]:
|
||||
"""Function to determine given an event defintion, if this handler can
|
||||
process it or not. This handler accepts events from watchdog with
|
||||
Python recipes"""
|
||||
@ -122,10 +122,10 @@ class PythonHandler(BaseHandler):
|
||||
valid_event(event)
|
||||
if event[EVENT_TYPE] == EVENT_TYPE_WATCHDOG \
|
||||
and type(event[EVENT_RULE].recipe) == PythonRecipe:
|
||||
return True
|
||||
except:
|
||||
return True, ""
|
||||
except Exception as e:
|
||||
pass
|
||||
return False
|
||||
return False, str(e)
|
||||
|
||||
def _is_valid_handler_base(self, handler_base)->None:
|
||||
"""Validation check for 'handler_base' variable from main
|
||||
@ -143,7 +143,7 @@ class PythonHandler(BaseHandler):
|
||||
meow_job = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
event,
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:yaml_dict,
|
||||
JOB_HASH: event[WATCHDOG_HASH],
|
||||
PYTHON_FUNC:job_func,
|
||||
|
@ -6,8 +6,8 @@ from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, EVENT_TYPE_WATCHDOG,
|
||||
WATCHDOG_BASE, EVENT_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \
|
||||
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
|
||||
BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
|
||||
from core.functionality import get_file_hash, create_event, create_job, \
|
||||
make_dir, write_yaml, write_notebook
|
||||
from core.functionality import get_file_hash, create_watchdog_event, \
|
||||
create_job, make_dir, write_yaml, write_notebook
|
||||
from core.meow import create_rule
|
||||
from conductors import LocalPythonConductor
|
||||
from patterns import FileEventPattern
|
||||
@ -69,17 +69,13 @@ class MeowTests(unittest.TestCase):
|
||||
|
||||
job_dict = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
create_watchdog_event(
|
||||
file_path,
|
||||
rule,
|
||||
{
|
||||
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
||||
EVENT_RULE: rule,
|
||||
WATCHDOG_HASH: file_hash
|
||||
}
|
||||
TEST_MONITOR_BASE,
|
||||
file_hash
|
||||
),
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:params_dict,
|
||||
JOB_HASH: file_hash,
|
||||
PYTHON_FUNC:job_func,
|
||||
@ -146,17 +142,13 @@ class MeowTests(unittest.TestCase):
|
||||
|
||||
bad_job_dict = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
create_watchdog_event(
|
||||
file_path,
|
||||
rule,
|
||||
{
|
||||
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
||||
EVENT_RULE: rule,
|
||||
WATCHDOG_HASH: file_hash
|
||||
}
|
||||
TEST_MONITOR_BASE,
|
||||
file_hash
|
||||
),
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:params_dict,
|
||||
JOB_HASH: file_hash,
|
||||
PYTHON_FUNC:job_func,
|
||||
@ -178,17 +170,13 @@ class MeowTests(unittest.TestCase):
|
||||
# Ensure execution can continue after one failed job
|
||||
good_job_dict = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
create_watchdog_event(
|
||||
file_path,
|
||||
rule,
|
||||
{
|
||||
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
||||
EVENT_RULE: rule,
|
||||
WATCHDOG_HASH: file_hash
|
||||
}
|
||||
TEST_MONITOR_BASE,
|
||||
file_hash
|
||||
),
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:params_dict,
|
||||
JOB_HASH: file_hash,
|
||||
PYTHON_FUNC:job_func,
|
||||
@ -249,17 +237,13 @@ class MeowTests(unittest.TestCase):
|
||||
|
||||
job_dict = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
create_watchdog_event(
|
||||
file_path,
|
||||
rule,
|
||||
{
|
||||
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
||||
EVENT_RULE: rule,
|
||||
WATCHDOG_HASH: file_hash
|
||||
}
|
||||
TEST_MONITOR_BASE,
|
||||
file_hash
|
||||
),
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:{
|
||||
"extra":"extra",
|
||||
"infile":file_path,
|
||||
|
@ -16,6 +16,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
||||
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
|
||||
make_dir, parameterize_jupyter_notebook, create_event, create_job, \
|
||||
replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \
|
||||
create_watchdog_event, create_fake_watchdog_event, \
|
||||
KEYWORD_PATH, KEYWORD_REL_PATH, KEYWORD_DIR, KEYWORD_REL_DIR, \
|
||||
KEYWORD_FILENAME, KEYWORD_PREFIX, KEYWORD_BASE, KEYWORD_EXTENSION, \
|
||||
KEYWORD_JOB
|
||||
@ -266,7 +267,7 @@ class CorrectnessTests(unittest.TestCase):
|
||||
self.assertEqual(event[EVENT_PATH], "path")
|
||||
self.assertEqual(event[EVENT_RULE], rule)
|
||||
|
||||
event2 = create_event("test2", "path2", rule, {"a":1})
|
||||
event2 = create_event("test2", "path2", rule, extras={"a":1})
|
||||
|
||||
self.assertEqual(type(event2), dict)
|
||||
self.assertTrue(EVENT_TYPE in event2.keys())
|
||||
@ -298,7 +299,7 @@ class CorrectnessTests(unittest.TestCase):
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
"file_path",
|
||||
rule,
|
||||
{
|
||||
extras={
|
||||
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
||||
EVENT_RULE: rule,
|
||||
WATCHDOG_HASH: "file_hash"
|
||||
@ -308,7 +309,7 @@ class CorrectnessTests(unittest.TestCase):
|
||||
job_dict = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
event,
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:{
|
||||
"extra":"extra",
|
||||
"infile":"file_path",
|
||||
@ -560,3 +561,101 @@ class CorrectnessTests(unittest.TestCase):
|
||||
self.assertFalse(os.path.exists(os.path.join(TEST_MONITOR_BASE, "A")))
|
||||
self.assertFalse(os.path.exists(
|
||||
os.path.join(TEST_MONITOR_BASE, "A", "B")))
|
||||
|
||||
def testCreateWatchdogEvent(self)->None:
|
||||
pattern = FileEventPattern(
|
||||
"pattern",
|
||||
"file_path",
|
||||
"recipe_one",
|
||||
"infile",
|
||||
parameters={
|
||||
"extra":"A line from a test Pattern",
|
||||
"outfile":"result_path"
|
||||
})
|
||||
recipe = JupyterNotebookRecipe(
|
||||
"recipe_one", APPENDING_NOTEBOOK)
|
||||
|
||||
rule = create_rule(pattern, recipe)
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
event = create_watchdog_event("path", rule)
|
||||
|
||||
event = create_watchdog_event("path", rule, "base", "hash")
|
||||
|
||||
self.assertEqual(type(event), dict)
|
||||
self.assertEqual(len(event.keys()), 5)
|
||||
self.assertTrue(EVENT_TYPE in event.keys())
|
||||
self.assertTrue(EVENT_PATH in event.keys())
|
||||
self.assertTrue(EVENT_RULE in event.keys())
|
||||
self.assertTrue(WATCHDOG_BASE in event.keys())
|
||||
self.assertTrue(WATCHDOG_HASH in event.keys())
|
||||
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
|
||||
self.assertEqual(event[EVENT_PATH], "path")
|
||||
self.assertEqual(event[EVENT_RULE], rule)
|
||||
self.assertEqual(event[WATCHDOG_BASE], "base")
|
||||
self.assertEqual(event[WATCHDOG_HASH], "hash")
|
||||
|
||||
event = create_watchdog_event(
|
||||
"path2", rule, "base", "hash", extras={"a":1}
|
||||
)
|
||||
|
||||
self.assertEqual(type(event), dict)
|
||||
self.assertTrue(EVENT_TYPE in event.keys())
|
||||
self.assertTrue(EVENT_PATH in event.keys())
|
||||
self.assertTrue(EVENT_RULE in event.keys())
|
||||
self.assertTrue(WATCHDOG_BASE in event.keys())
|
||||
self.assertTrue(WATCHDOG_HASH in event.keys())
|
||||
self.assertEqual(len(event.keys()), 6)
|
||||
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
|
||||
self.assertEqual(event[EVENT_PATH], "path2")
|
||||
self.assertEqual(event[EVENT_RULE], rule)
|
||||
self.assertEqual(event["a"], 1)
|
||||
self.assertEqual(event[WATCHDOG_BASE], "base")
|
||||
self.assertEqual(event[WATCHDOG_HASH], "hash")
|
||||
|
||||
def testCreateFakeWatchdogEvent(self)->None:
|
||||
pattern = FileEventPattern(
|
||||
"pattern",
|
||||
"file_path",
|
||||
"recipe_one",
|
||||
"infile",
|
||||
parameters={
|
||||
"extra":"A line from a test Pattern",
|
||||
"outfile":"result_path"
|
||||
})
|
||||
recipe = JupyterNotebookRecipe(
|
||||
"recipe_one", APPENDING_NOTEBOOK)
|
||||
|
||||
rule = create_rule(pattern, recipe)
|
||||
|
||||
with self.assertRaises(TypeError):
|
||||
event = create_fake_watchdog_event("path", rule)
|
||||
|
||||
event = create_fake_watchdog_event("path", rule, "base")
|
||||
|
||||
self.assertEqual(type(event), dict)
|
||||
self.assertEqual(len(event.keys()), 4)
|
||||
self.assertTrue(EVENT_TYPE in event.keys())
|
||||
self.assertTrue(EVENT_PATH in event.keys())
|
||||
self.assertTrue(EVENT_RULE in event.keys())
|
||||
self.assertTrue(WATCHDOG_BASE in event.keys())
|
||||
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
|
||||
self.assertEqual(event[EVENT_PATH], "path")
|
||||
self.assertEqual(event[EVENT_RULE], rule)
|
||||
self.assertEqual(event[WATCHDOG_BASE], "base")
|
||||
|
||||
event = create_fake_watchdog_event(
|
||||
"path2", rule, "base", extras={"a":1}
|
||||
)
|
||||
|
||||
self.assertEqual(type(event), dict)
|
||||
self.assertTrue(EVENT_TYPE in event.keys())
|
||||
self.assertTrue(EVENT_PATH in event.keys())
|
||||
self.assertTrue(EVENT_RULE in event.keys())
|
||||
self.assertTrue(WATCHDOG_BASE in event.keys())
|
||||
self.assertEqual(len(event.keys()), 5)
|
||||
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
|
||||
self.assertEqual(event[EVENT_PATH], "path2")
|
||||
self.assertEqual(event[EVENT_RULE], rule)
|
||||
self.assertEqual(event["a"], 1)
|
||||
self.assertEqual(event[WATCHDOG_BASE], "base")
|
||||
|
@ -1,7 +1,7 @@
|
||||
|
||||
import unittest
|
||||
|
||||
from typing import Any, Union
|
||||
from typing import Any, Union, Tuple
|
||||
|
||||
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
|
||||
BaseHandler, BaseConductor, create_rules, create_rule
|
||||
@ -198,7 +198,8 @@ class MeowTests(unittest.TestCase):
|
||||
pass
|
||||
def _is_valid_inputs(self, inputs:Any)->None:
|
||||
pass
|
||||
def valid_handle_criteria(self, event:dict[str,Any])->bool:
|
||||
def valid_handle_criteria(self, event:dict[str,Any]
|
||||
)->Tuple[bool,str]:
|
||||
pass
|
||||
|
||||
FullTestHandler()
|
||||
@ -218,7 +219,8 @@ class MeowTests(unittest.TestCase):
|
||||
def execute(self, job:dict[str,Any])->None:
|
||||
pass
|
||||
|
||||
def valid_execute_criteria(self, job:dict[str,Any])->bool:
|
||||
def valid_execute_criteria(self, job:dict[str,Any]
|
||||
)->Tuple[bool,str]:
|
||||
pass
|
||||
|
||||
FullTestConductor()
|
||||
|
@ -11,8 +11,8 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
|
||||
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \
|
||||
PARAMS_FILE, JOB_FILE, RESULT_FILE
|
||||
from core.correctness.validation import valid_job
|
||||
from core.functionality import get_file_hash, create_job, create_event, \
|
||||
make_dir, write_yaml, write_notebook, read_yaml
|
||||
from core.functionality import get_file_hash, create_job, \
|
||||
create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml
|
||||
from core.meow import create_rules, create_rule
|
||||
from patterns.file_event_pattern import FileEventPattern, SWEEP_START, \
|
||||
SWEEP_STOP, SWEEP_JUMP
|
||||
@ -351,17 +351,13 @@ class JupyterNotebookTests(unittest.TestCase):
|
||||
|
||||
job_dict = create_job(
|
||||
JOB_TYPE_PYTHON,
|
||||
create_event(
|
||||
EVENT_TYPE_WATCHDOG,
|
||||
create_watchdog_event(
|
||||
file_path,
|
||||
rule,
|
||||
{
|
||||
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
||||
EVENT_RULE: rule,
|
||||
WATCHDOG_HASH: file_hash
|
||||
}
|
||||
TEST_MONITOR_BASE,
|
||||
file_hash
|
||||
),
|
||||
{
|
||||
extras={
|
||||
JOB_PARAMETERS:params_dict,
|
||||
JOB_HASH: file_hash,
|
||||
PYTHON_FUNC:job_func,
|
||||
|
@ -9,10 +9,11 @@ from typing import Any, Union
|
||||
from core.correctness.validation import check_type, check_implementation, \
|
||||
valid_string, valid_dict, valid_list, valid_existing_file_path, \
|
||||
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
|
||||
setup_debugging
|
||||
setup_debugging, valid_watchdog_event
|
||||
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
|
||||
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
|
||||
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE
|
||||
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
|
||||
WATCHDOG_HASH
|
||||
from core.functionality import make_dir
|
||||
from shared import setup, teardown, TEST_MONITOR_BASE
|
||||
|
||||
@ -303,4 +304,36 @@ class CorrectnessTests(unittest.TestCase):
|
||||
with self.assertRaises(TypeError):
|
||||
setup_debugging(stream, "1")
|
||||
|
||||
#TODO test watchdog event dict
|
||||
#Test watchdog event dict
|
||||
def testWatchdogEventValidation(self)->None:
|
||||
valid_watchdog_event({
|
||||
EVENT_TYPE: "test",
|
||||
EVENT_PATH: "path",
|
||||
EVENT_RULE: "rule",
|
||||
WATCHDOG_HASH: "hash",
|
||||
WATCHDOG_BASE: "base"
|
||||
})
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
valid_watchdog_event({
|
||||
EVENT_TYPE: "test",
|
||||
EVENT_PATH: "path",
|
||||
EVENT_RULE: "rule"
|
||||
})
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
valid_watchdog_event({
|
||||
EVENT_TYPE: "anything",
|
||||
EVENT_PATH: "path",
|
||||
EVENT_RULE: "rule",
|
||||
"a": 1
|
||||
})
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
valid_event({EVENT_TYPE: "test"})
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
valid_event({"EVENT_TYPE": "test"})
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
valid_event({})
|
Reference in New Issue
Block a user