NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

standardised event construction and validation a bit more

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-02 17:41:18 +01:00
parent 636d70f4e8
commit 64aaf46196
13 changed files with 243 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
tests/test_conductors.py
View File

@ -6,8 +6,8 @@ from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, EVENT_TYPE_WATCHDOG,
WATCHDOG_BASE, EVENT_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
from core.functionality import get_file_hash, create_event, create_job, \
make_dir, write_yaml, write_notebook
from core.functionality import get_file_hash, create_watchdog_event, \
create_job, make_dir, write_yaml, write_notebook
from core.meow import create_rule
from conductors import LocalPythonConductor
from patterns import FileEventPattern
@ -69,17 +69,13 @@ class MeowTests(unittest.TestCase):
job_dict = create_job(
JOB_TYPE_PYTHON,
create_event(
EVENT_TYPE_WATCHDOG,
create_watchdog_event(
file_path,
rule,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: file_hash
}
TEST_MONITOR_BASE,
file_hash
),
{
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
@ -146,17 +142,13 @@ class MeowTests(unittest.TestCase):
bad_job_dict = create_job(
JOB_TYPE_PYTHON,
create_event(
EVENT_TYPE_WATCHDOG,
create_watchdog_event(
file_path,
rule,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: file_hash
}
TEST_MONITOR_BASE,
file_hash
),
{
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
@ -178,17 +170,13 @@ class MeowTests(unittest.TestCase):
# Ensure execution can continue after one failed job
good_job_dict = create_job(
JOB_TYPE_PYTHON,
create_event(
EVENT_TYPE_WATCHDOG,
create_watchdog_event(
file_path,
rule,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: file_hash
}
TEST_MONITOR_BASE,
file_hash
),
{
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
@ -249,17 +237,13 @@ class MeowTests(unittest.TestCase):
job_dict = create_job(
JOB_TYPE_PYTHON,
create_event(
EVENT_TYPE_WATCHDOG,
create_watchdog_event(
file_path,
rule,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: file_hash
}
TEST_MONITOR_BASE,
file_hash
),
{
extras={
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,

105
tests/test_functionality.py
View File

@ -16,6 +16,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
make_dir, parameterize_jupyter_notebook, create_event, create_job, \
replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \
create_watchdog_event, create_fake_watchdog_event, \
KEYWORD_PATH, KEYWORD_REL_PATH, KEYWORD_DIR, KEYWORD_REL_DIR, \
KEYWORD_FILENAME, KEYWORD_PREFIX, KEYWORD_BASE, KEYWORD_EXTENSION, \
KEYWORD_JOB
@ -266,7 +267,7 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual(event[EVENT_PATH], "path")
self.assertEqual(event[EVENT_RULE], rule)
event2 = create_event("test2", "path2", rule, {"a":1})
event2 = create_event("test2", "path2", rule, extras={"a":1})
self.assertEqual(type(event2), dict)
self.assertTrue(EVENT_TYPE in event2.keys())
@ -298,7 +299,7 @@ class CorrectnessTests(unittest.TestCase):
EVENT_TYPE_WATCHDOG,
"file_path",
rule,
{
extras={
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: "file_hash"
@ -308,7 +309,7 @@ class CorrectnessTests(unittest.TestCase):
job_dict = create_job(
JOB_TYPE_PYTHON,
event,
{
extras={
JOB_PARAMETERS:{
"extra":"extra",
"infile":"file_path",
@ -560,3 +561,101 @@ class CorrectnessTests(unittest.TestCase):
self.assertFalse(os.path.exists(os.path.join(TEST_MONITOR_BASE, "A")))
self.assertFalse(os.path.exists(
os.path.join(TEST_MONITOR_BASE, "A", "B")))
def testCreateWatchdogEvent(self)->None:
pattern = FileEventPattern(
"pattern",
"file_path",
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"result_path"
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
with self.assertRaises(TypeError):
event = create_watchdog_event("path", rule)
event = create_watchdog_event("path", rule, "base", "hash")
self.assertEqual(type(event), dict)
self.assertEqual(len(event.keys()), 5)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertTrue(WATCHDOG_HASH in event.keys())
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event[WATCHDOG_BASE], "base")
self.assertEqual(event[WATCHDOG_HASH], "hash")
event = create_watchdog_event(
"path2", rule, "base", "hash", extras={"a":1}
)
self.assertEqual(type(event), dict)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertTrue(WATCHDOG_HASH in event.keys())
self.assertEqual(len(event.keys()), 6)
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path2")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event["a"], 1)
self.assertEqual(event[WATCHDOG_BASE], "base")
self.assertEqual(event[WATCHDOG_HASH], "hash")
def testCreateFakeWatchdogEvent(self)->None:
pattern = FileEventPattern(
"pattern",
"file_path",
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"result_path"
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
with self.assertRaises(TypeError):
event = create_fake_watchdog_event("path", rule)
event = create_fake_watchdog_event("path", rule, "base")
self.assertEqual(type(event), dict)
self.assertEqual(len(event.keys()), 4)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event[WATCHDOG_BASE], "base")
event = create_fake_watchdog_event(
"path2", rule, "base", extras={"a":1}
)
self.assertEqual(type(event), dict)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertEqual(len(event.keys()), 5)
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path2")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event["a"], 1)
self.assertEqual(event[WATCHDOG_BASE], "base")

8
tests/test_meow.py
View File

@ -1,7 +1,7 @@
import unittest
from typing import Any, Union
from typing import Any, Union, Tuple
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, BaseConductor, create_rules, create_rule
@ -198,7 +198,8 @@ class MeowTests(unittest.TestCase):
pass
def _is_valid_inputs(self, inputs:Any)->None:
pass
def valid_handle_criteria(self, event:dict[str,Any])->bool:
def valid_handle_criteria(self, event:dict[str,Any]
)->Tuple[bool,str]:
pass
FullTestHandler()
@ -218,7 +219,8 @@ class MeowTests(unittest.TestCase):
def execute(self, job:dict[str,Any])->None:
pass
def valid_execute_criteria(self, job:dict[str,Any])->bool:
def valid_execute_criteria(self, job:dict[str,Any]
)->Tuple[bool,str]:
pass
FullTestConductor()

16
tests/test_recipes.py
View File

@ -11,8 +11,8 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \
PARAMS_FILE, JOB_FILE, RESULT_FILE
from core.correctness.validation import valid_job
from core.functionality import get_file_hash, create_job, create_event, \
make_dir, write_yaml, write_notebook, read_yaml
from core.functionality import get_file_hash, create_job, \
create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml
from core.meow import create_rules, create_rule
from patterns.file_event_pattern import FileEventPattern, SWEEP_START, \
SWEEP_STOP, SWEEP_JUMP
@ -351,17 +351,13 @@ class JupyterNotebookTests(unittest.TestCase):
job_dict = create_job(
JOB_TYPE_PYTHON,
create_event(
EVENT_TYPE_WATCHDOG,
create_watchdog_event(
file_path,
rule,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: file_hash
}
TEST_MONITOR_BASE,
file_hash
),
{
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,

39
tests/test_validation.py
View File

@ -9,10 +9,11 @@ from typing import Any, Union
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
setup_debugging
setup_debugging, valid_watchdog_event
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
WATCHDOG_HASH
from core.functionality import make_dir
from shared import setup, teardown, TEST_MONITOR_BASE
@ -303,4 +304,36 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(TypeError):
setup_debugging(stream, "1")
#TODO test watchdog event dict
#Test watchdog event dict
def testWatchdogEventValidation(self)->None:
valid_watchdog_event({
EVENT_TYPE: "test",
EVENT_PATH: "path",
EVENT_RULE: "rule",
WATCHDOG_HASH: "hash",
WATCHDOG_BASE: "base"
})
with self.assertRaises(KeyError):
valid_watchdog_event({
EVENT_TYPE: "test",
EVENT_PATH: "path",
EVENT_RULE: "rule"
})
with self.assertRaises(KeyError):
valid_watchdog_event({
EVENT_TYPE: "anything",
EVENT_PATH: "path",
EVENT_RULE: "rule",
"a": 1
})
with self.assertRaises(KeyError):
valid_event({EVENT_TYPE: "test"})
with self.assertRaises(KeyError):
valid_event({"EVENT_TYPE": "test"})
with self.assertRaises(KeyError):
valid_event({})