NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added time to all events. this is a unix timestamp so will need to be converted to something nicer if dispalyed, but hey, its easy to store

Browse Source
This commit is contained in:
PatchOfScotland
2023-04-27 15:13:47 +02:00
parent d3eb2dbf9f
commit 933d568fb2
13 changed files with 210 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/base_handler.py
View File

@ -14,9 +14,10 @@ from typing import Any, Tuple, Dict, Union
from time import sleep
from meow_base.core.vars import VALID_CHANNELS, EVENT_RULE, EVENT_PATH, \
VALID_HANDLER_NAME_CHARS, META_FILE, JOB_ID, WATCHDOG_BASE, JOB_FILE, \
JOB_PARAMETERS, get_drt_imp_msg
VALID_HANDLER_NAME_CHARS, META_FILE, JOB_ID, JOB_FILE, JOB_PARAMETERS, \
get_drt_imp_msg
from meow_base.core.meow import valid_event
from meow_base.patterns.file_event_pattern import WATCHDOG_HASH
from meow_base.functionality.file_io import threadsafe_write_status, \
threadsafe_update_status, make_dir, write_file, lines_to_string
from meow_base.functionality.validation import check_implementation, \
@ -179,6 +180,7 @@ class BaseHandler:
# Get updated job parameters
# TODO replace this with generic implementation
from meow_base.patterns.file_event_pattern import WATCHDOG_BASE
params_dict = replace_keywords(
params_dict,
meow_job[JOB_ID],
@ -242,8 +244,8 @@ class BaseHandler:
"#!/bin/bash",
"",
"# Get job params",
"given_hash=$(grep 'file_hash: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)",
"event_path=$(grep 'event_path: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)",
f"given_hash=$(grep '{WATCHDOG_HASH}: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)",
f"event_path=$(grep '{EVENT_PATH}: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)",
"",
"echo event_path: $event_path",
"echo given_hash: $given_hash",

12
core/meow.py
View File

@ -12,21 +12,16 @@ from meow_base.core.rule import Rule
from meow_base.functionality.validation import check_type
from meow_base.core.vars import EVENT_TYPE, EVENT_PATH, \
JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, EVENT_TIME
# Required keys in event dict
EVENT_KEYS = {
EVENT_TYPE: str,
EVENT_PATH: str,
EVENT_TIME: float,
EVENT_RULE: Rule
}
WATCHDOG_EVENT_KEYS = {
WATCHDOG_BASE: str,
WATCHDOG_HASH: str,
**EVENT_KEYS
}
# Required keys in job dict
JOB_KEYS = {
JOB_TYPE: str,
@ -59,6 +54,3 @@ def valid_event(event:Dict[str,Any])->None:
def valid_job(job:Dict[str,Any])->None:
"""Check that a given dict expresses a meow job."""
valid_meow_dict(job, "Job", JOB_KEYS)
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

8
core/vars.py
View File

@ -44,12 +44,8 @@ SHA256 = "sha256"
# meow events
EVENT_TYPE = "event_type"
EVENT_PATH = "event_path"
EVENT_RULE = "rule"
# watchdog events
EVENT_TYPE_WATCHDOG = "watchdog"
WATCHDOG_BASE = "monitor_base"
WATCHDOG_HASH = "file_hash"
EVENT_RULE = "event_rule"
EVENT_TIME = "event_time"
# inotify events
FILE_CREATE_EVENT = "file_created"

30
functionality/meow.py
View File

@ -13,11 +13,10 @@ from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.rule import Rule
from meow_base.functionality.validation import check_type, valid_dict, \
valid_list
from meow_base.core.vars import EVENT_PATH, EVENT_RULE, \
EVENT_TYPE, EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \
from meow_base.core.vars import EVENT_PATH, EVENT_RULE, EVENT_TIME, \
EVENT_TYPE, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \
JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
JOB_TYPE, STATUS_CREATING, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \
SWEEP_START, SWEEP_STOP
JOB_TYPE, STATUS_CREATING, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
from meow_base.functionality.naming import generate_job_id
# mig trigger keyword replacements
@ -104,32 +103,17 @@ def create_parameter_sweep(variable_name:str, start:Union[int,float,complex],
}
}
def create_event(event_type:str, path:str, rule:Any, extras:Dict[Any,Any]={}
)->Dict[Any,Any]:
def create_event(event_type:str, path:str, rule:Any, time:float,
extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW dictionary."""
return {
**extras,
EVENT_PATH: path,
EVENT_TYPE: event_type,
EVENT_RULE: rule
EVENT_RULE: rule,
EVENT_TIME: time
}
def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW event dictionary."""
return create_event(
EVENT_TYPE_WATCHDOG,
path,
rule,
extras={
**extras,
**{
WATCHDOG_HASH: hash,
WATCHDOG_BASE: base
}
}
)
def create_job_metadata_dict(job_type:str, event:Dict[str,Any],
extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW job dictionary."""

37
patterns/file_event_pattern.py
View File

@ -20,6 +20,7 @@ from watchdog.events import PatternMatchingEventHandler
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.meow import EVENT_KEYS, valid_meow_dict
from meow_base.core.rule import Rule
from meow_base.functionality.validation import check_type, valid_string, \
valid_dict, valid_list, valid_dir_path
@ -30,7 +31,7 @@ from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \
DIR_RETROACTIVE_EVENT
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.hashing import get_hash
from meow_base.functionality.meow import create_watchdog_event
from meow_base.functionality.meow import create_event
# Events that are monitored by default
_DEFAULT_MASK = [
@ -41,6 +42,38 @@ _DEFAULT_MASK = [
FILE_CLOSED_EVENT
]
# watchdog events
EVENT_TYPE_WATCHDOG = "watchdog"
WATCHDOG_BASE = "monitor_base"
WATCHDOG_HASH = "file_hash"
WATCHDOG_EVENT_KEYS = {
WATCHDOG_BASE: str,
WATCHDOG_HASH: str,
**EVENT_KEYS
}
def create_watchdog_event(path:str, rule:Any, base:str, time:float,
hash:str, extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW event dictionary."""
return create_event(
EVENT_TYPE_WATCHDOG,
path,
rule,
time,
extras={
**extras,
**{
WATCHDOG_HASH: hash,
WATCHDOG_BASE: base
}
}
)
def valid_watchdog_event(event:Dict[str,Any])->None:
valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)
class FileEventPattern(BasePattern):
# The path at which events will trigger this pattern
triggering_path:str
@ -239,6 +272,7 @@ class WatchdogMonitor(BaseMonitor):
event.src_path,
rule,
self.base_dir,
event.time_stamp,
get_hash(event.src_path, SHA256)
)
print_debug(self._print_target, self.debug_level,
@ -300,6 +334,7 @@ class WatchdogMonitor(BaseMonitor):
globble,
rule,
self.base_dir,
time(),
get_hash(globble, SHA256)
)
print_debug(self._print_target, self.debug_level,

4
recipes/bash_recipe.py
View File

@ -11,13 +11,13 @@ from meow_base.core.meow import valid_event
from meow_base.functionality.validation import check_type, valid_dict, \
valid_string, valid_dir_path
from meow_base.core.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
VALID_VARIABLE_NAME_CHARS, EVENT_RULE, EVENT_TYPE, EVENT_TYPE_WATCHDOG, \
VALID_VARIABLE_NAME_CHARS, EVENT_RULE, EVENT_TYPE, \
JOB_TYPE_BASH
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import valid_path, make_dir, write_file, \
lines_to_string
from meow_base.functionality.parameterisation import parameterize_bash_script
from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG
class BashRecipe(BaseRecipe):
# A path to the bash script used to create this recipe

3
recipes/jupyter_notebook_recipe.py
View File

@ -18,13 +18,14 @@ from meow_base.core.meow import valid_event
from meow_base.functionality.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path, valid_existing_file_path
from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, DEFAULT_JOB_QUEUE_DIR, \
DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
JOB_TYPE_PAPERMILL, EVENT_RULE, EVENT_TYPE, EVENT_RULE
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_notebook, \
write_notebook
from meow_base.functionality.parameterisation import \
parameterize_jupyter_notebook
from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG
class JupyterNotebookRecipe(BaseRecipe):
# A path to the jupyter notebook used to create this recipe

4
recipes/python_recipe.py
View File

@ -17,13 +17,13 @@ from meow_base.core.meow import valid_event
from meow_base.functionality.validation import check_script, valid_string, \
valid_dict, valid_dir_path
from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, \
DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, \
JOB_TYPE_PYTHON, EVENT_TYPE, EVENT_RULE
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, write_file, \
lines_to_string
from meow_base.functionality.parameterisation import parameterize_python_script
from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG
class PythonRecipe(BaseRecipe):
def __init__(self, name:str, recipe:List[str], parameters:Dict[str,Any]={},

18
tests/test_conductors.py
View File

@ -5,6 +5,7 @@ import unittest
from datetime import datetime
from multiprocessing import Pipe
from time import time
from typing import Dict
from meow_base.core.vars import JOB_TYPE_PYTHON, SHA256, \
@ -12,15 +13,16 @@ from meow_base.core.vars import JOB_TYPE_PYTHON, SHA256, \
JOB_EVENT, META_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \
JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \
JOB_CREATE_TIME, JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_FILE
JOB_TYPE_BASH, JOB_FILE
from meow_base.conductors import LocalPythonConductor, LocalBashConductor
from meow_base.functionality.file_io import read_file, read_yaml, write_file, \
write_yaml, lines_to_string, make_dir, threadsafe_read_status
from meow_base.functionality.hashing import get_hash
from meow_base.functionality.meow import create_watchdog_event, create_job_metadata_dict, \
from meow_base.functionality.meow import create_job_metadata_dict, \
create_rule
from meow_base.functionality.parameterisation import parameterize_bash_script
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.patterns.file_event_pattern import FileEventPattern, \
EVENT_TYPE_WATCHDOG, create_watchdog_event
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler
from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler
@ -101,6 +103,7 @@ class PythonTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
)
@ -192,6 +195,7 @@ class PythonTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
)
@ -268,6 +272,7 @@ class PythonTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
),
extras={
@ -333,6 +338,7 @@ class PythonTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
),
extras={
@ -401,6 +407,7 @@ class PythonTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
),
extras={
@ -598,6 +605,7 @@ class BashTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
)
@ -689,6 +697,7 @@ class BashTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
),
extras={
@ -755,6 +764,7 @@ class BashTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
),
extras={
@ -839,6 +849,7 @@ class BashTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
),
extras={
@ -908,6 +919,7 @@ class BashTests(unittest.TestCase):
file_path,
rule,
TEST_MONITOR_BASE,
time(),
file_hash
),
extras={

88
tests/test_functionality.py
View File

@ -8,16 +8,16 @@ from datetime import datetime
from multiprocessing import Pipe, Queue
from os.path import basename
from sys import prefix, base_prefix
from time import sleep
from time import sleep, time
from typing import Dict
from meow_base.core.meow import EVENT_KEYS
from meow_base.core.rule import Rule
from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, LOCK_EXT, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, \
SHA256, EVENT_TYPE, EVENT_PATH, LOCK_EXT, EVENT_RULE, JOB_PARAMETERS, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, JOB_ERROR, STATUS_DONE, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL, STATUS_CREATING
JOB_REQUIREMENTS, JOB_TYPE_PAPERMILL, STATUS_CREATING
from meow_base.functionality.debug import setup_debugging
from meow_base.functionality.file_io import lines_to_string, make_dir, \
read_file, read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
@ -28,7 +28,7 @@ from meow_base.functionality.meow import KEYWORD_BASE, KEYWORD_DIR, \
KEYWORD_EXTENSION, KEYWORD_FILENAME, KEYWORD_JOB, KEYWORD_PATH, \
KEYWORD_PREFIX, KEYWORD_REL_DIR, KEYWORD_REL_PATH, \
create_event, create_job_metadata_dict, create_rule, create_rules, \
create_watchdog_event, replace_keywords, create_parameter_sweep
replace_keywords, create_parameter_sweep
from meow_base.functionality.naming import _generate_id
from meow_base.functionality.parameterisation import \
parameterize_jupyter_notebook, parameterize_python_script
@ -36,7 +36,8 @@ from meow_base.functionality.process_io import wait
from meow_base.functionality.requirements import REQUIREMENT_PYTHON, \
REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, REQ_PYTHON_VERSION, \
create_python_requirements, check_requirements
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.patterns.file_event_pattern import FileEventPattern, \
EVENT_TYPE_WATCHDOG, WATCHDOG_BASE, WATCHDOG_HASH
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, \
COMPLETE_PYTHON_SCRIPT, valid_recipe_two, valid_recipe_one, \
@ -622,24 +623,26 @@ class MeowTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
event = create_event("test", "path", rule)
event = create_event("test", "path", rule, time())
self.assertEqual(type(event), dict)
self.assertEqual(len(event.keys()), 3)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertEqual(len(event.keys()), len(EVENT_KEYS))
for key, value in EVENT_KEYS.items():
self.assertTrue(key in event.keys())
self.assertIsInstance(event[key], value)
self.assertEqual(event[EVENT_TYPE], "test")
self.assertEqual(event[EVENT_PATH], "path")
self.assertEqual(event[EVENT_RULE], rule)
event2 = create_event("test2", "path2", rule, extras={"a":1})
event2 = create_event(
"test2", "path2", rule, time(), extras={"a":1}
)
self.assertEqual(type(event2), dict)
self.assertTrue(EVENT_TYPE in event2.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertEqual(len(event2.keys()), 4)
self.assertEqual(len(event.keys()), len(EVENT_KEYS))
for key, value in EVENT_KEYS.items():
self.assertTrue(key in event.keys())
self.assertIsInstance(event[key], value)
self.assertEqual(event2[EVENT_TYPE], "test2")
self.assertEqual(event2[EVENT_PATH], "path2")
self.assertEqual(event2[EVENT_RULE], rule)
@ -665,6 +668,7 @@ class MeowTests(unittest.TestCase):
EVENT_TYPE_WATCHDOG,
"file_path",
rule,
time(),
extras={
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
@ -705,58 +709,6 @@ class MeowTests(unittest.TestCase):
self.assertIn(JOB_REQUIREMENTS, job_dict)
self.assertEqual(job_dict[JOB_REQUIREMENTS], {})
# Test creation of watchdog event dict
def testCreateWatchdogEvent(self)->None:
pattern = FileEventPattern(
"pattern",
"file_path",
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"result_path"
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
with self.assertRaises(TypeError):
event = create_watchdog_event("path", rule)
event = create_watchdog_event("path", rule, "base", "hash")
self.assertEqual(type(event), dict)
self.assertEqual(len(event.keys()), 5)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertTrue(WATCHDOG_HASH in event.keys())
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event[WATCHDOG_BASE], "base")
self.assertEqual(event[WATCHDOG_HASH], "hash")
event = create_watchdog_event(
"path2", rule, "base", "hash", extras={"a":1}
)
self.assertEqual(type(event), dict)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertTrue(WATCHDOG_HASH in event.keys())
self.assertEqual(len(event.keys()), 6)
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path2")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event["a"], 1)
self.assertEqual(event[WATCHDOG_BASE], "base")
self.assertEqual(event[WATCHDOG_HASH], "hash")
# Test that replace_keywords replaces MEOW keywords in a given dictionary
def testReplaceKeywords(self)->None:
test_dict = {

72
tests/test_patterns.py
View File

@ -3,19 +3,22 @@ import io
import os
import unittest
from datetime import datetime
from multiprocessing import Pipe
from time import sleep
from time import sleep, time
from meow_base.core.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
EVENT_RULE, EVENT_PATH, SWEEP_START, \
SWEEP_JUMP, SWEEP_STOP, DIR_EVENTS
from meow_base.functionality.file_io import make_dir
from meow_base.functionality.meow import create_rule
from meow_base.patterns.file_event_pattern import FileEventPattern, \
WatchdogMonitor, _DEFAULT_MASK
WatchdogMonitor, _DEFAULT_MASK, WATCHDOG_HASH, WATCHDOG_BASE, \
EVENT_TYPE_WATCHDOG, WATCHDOG_EVENT_KEYS, create_watchdog_event
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from meow_base.recipes.python_recipe import PythonRecipe
from shared import BAREBONES_NOTEBOOK, TEST_MONITOR_BASE, \
COUNTING_PYTHON_SCRIPT, setup, teardown
COUNTING_PYTHON_SCRIPT, APPENDING_NOTEBOOK, setup, teardown
def patterns_equal(tester, pattern_one, pattern_two):
@ -196,6 +199,67 @@ class WatchdogMonitorTests(unittest.TestCase):
super().tearDown()
teardown()
# Test creation of watchdog event dict
def testCreateWatchdogEvent(self)->None:
pattern = FileEventPattern(
"pattern",
"file_path",
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"result_path"
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
with self.assertRaises(TypeError):
event = create_watchdog_event("path", rule)
event = create_watchdog_event(
"path", rule, "base", time(), "hash")
self.assertEqual(type(event), dict)
self.assertEqual(len(event.keys()), len(WATCHDOG_EVENT_KEYS))
for key, value in WATCHDOG_EVENT_KEYS.items():
self.assertTrue(key in event.keys())
self.assertIsInstance(event[key], value)
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event[WATCHDOG_BASE], "base")
self.assertEqual(event[WATCHDOG_HASH], "hash")
event = create_watchdog_event(
"path2",
rule,
"base",
time(),
"hash",
extras={"a":1}
)
self.assertEqual(type(event), dict)
self.assertTrue(EVENT_TYPE in event.keys())
self.assertTrue(EVENT_PATH in event.keys())
self.assertTrue(EVENT_RULE in event.keys())
self.assertTrue(WATCHDOG_BASE in event.keys())
self.assertTrue(WATCHDOG_HASH in event.keys())
self.assertEqual(len(event.keys()), len(WATCHDOG_EVENT_KEYS)+1)
for key, value in WATCHDOG_EVENT_KEYS.items():
self.assertTrue(key in event.keys())
self.assertIsInstance(event[key], value)
self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
self.assertEqual(event[EVENT_PATH], "path2")
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event["a"], 1)
self.assertEqual(event[WATCHDOG_BASE], "base")
self.assertEqual(event[WATCHDOG_HASH], "hash")
#TODO test valid watchdog event
# Test WatchdogMonitor created
def testWatchdogMonitorMinimum(self)->None:
from_monitor = Pipe()

49
tests/test_recipes.py
View File

@ -4,18 +4,19 @@ import os
import unittest
from multiprocessing import Pipe
from time import time
from meow_base.core.meow import valid_job
from meow_base.core.vars import EVENT_TYPE, WATCHDOG_BASE, \
EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \
from meow_base.core.vars import EVENT_TYPE, EVENT_RULE, EVENT_PATH, SHA256, \
JOB_PARAMETERS, JOB_FILE, META_FILE, SWEEP_STOP, SWEEP_JUMP, \
SWEEP_START
SWEEP_START, EVENT_TIME
from meow_base.core.rule import Rule
from meow_base.functionality.file_io import read_yaml, write_notebook, \
threadsafe_read_status
from meow_base.functionality.hashing import get_hash
from meow_base.functionality.meow import create_rules, create_rule
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.patterns.file_event_pattern import FileEventPattern, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_TYPE_WATCHDOG
from meow_base.recipes.bash_recipe import BashRecipe, BashHandler
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, get_recipe_from_notebook
@ -163,6 +164,7 @@ class PapermillHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -214,6 +216,7 @@ class PapermillHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -284,6 +287,7 @@ class PapermillHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -348,21 +352,24 @@ class PapermillHandlerTests(unittest.TestCase):
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: "type",
EVENT_RULE: rule
EVENT_RULE: rule.name,
EVENT_TIME: time()
})
self.assertFalse(status)
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: "rule"
EVENT_RULE: "rule",
EVENT_TIME: time()
})
self.assertFalse(status)
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: rule
EVENT_RULE: rule,
EVENT_TIME: time()
})
self.assertTrue(status)
@ -432,6 +439,7 @@ class PapermillHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -577,6 +585,7 @@ class PythonHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -628,6 +637,7 @@ class PythonHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -698,6 +708,7 @@ class PythonHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -763,21 +774,24 @@ class PythonHandlerTests(unittest.TestCase):
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: "type",
EVENT_RULE: rule
EVENT_RULE: rule,
EVENT_TIME: time()
})
self.assertFalse(status)
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: "rule"
EVENT_RULE: "rule",
EVENT_TIME: time()
})
self.assertFalse(status)
status, s = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: rule
EVENT_RULE: rule,
EVENT_TIME: time()
})
self.assertTrue(status)
@ -834,6 +848,7 @@ class PythonHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -979,6 +994,7 @@ class BashHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -1030,6 +1046,7 @@ class BashHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -1100,6 +1117,7 @@ class BashHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -1176,6 +1194,7 @@ class BashHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
@ -1221,21 +1240,24 @@ class BashHandlerTests(unittest.TestCase):
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: "type",
EVENT_RULE: rule
EVENT_RULE: rule,
EVENT_TIME: time()
})
self.assertFalse(status)
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: "rule"
EVENT_RULE: "rule",
EVENT_TIME: time()
})
self.assertFalse(status)
status, s = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: rule
EVENT_RULE: rule,
EVENT_TIME: time()
})
self.assertTrue(status)
@ -1292,6 +1314,7 @@ class BashHandlerTests(unittest.TestCase):
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: get_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)

18
tests/test_validation.py
View File

@ -3,20 +3,21 @@ import unittest
import os
from datetime import datetime
from time import time
from typing import Any, Union
from meow_base.core.meow import valid_event, valid_job, \
valid_watchdog_event
from meow_base.core.meow import valid_event, valid_job
from meow_base.functionality.validation import check_type, \
check_implementation, valid_string, valid_dict, valid_list, \
valid_existing_file_path, valid_dir_path, valid_non_existing_path, \
check_callable, valid_natural, valid_dict_multiple_types
from meow_base.core.vars import VALID_NAME_CHARS, SHA256, \
EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
WATCHDOG_BASE, WATCHDOG_HASH
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, EVENT_TIME
from meow_base.functionality.file_io import make_dir
from meow_base.functionality.meow import create_rule
from meow_base.patterns.file_event_pattern import WATCHDOG_BASE, \
WATCHDOG_HASH, valid_watchdog_event
from shared import TEST_MONITOR_BASE, valid_pattern_one, valid_recipe_one, \
setup, teardown
@ -313,12 +314,14 @@ class MeowTests(unittest.TestCase):
valid_event({
EVENT_TYPE: "test",
EVENT_PATH: "path",
EVENT_RULE: rule
EVENT_RULE: rule,
EVENT_TIME: time()
})
valid_event({
EVENT_TYPE: "anything",
EVENT_PATH: "path",
EVENT_RULE: rule,
EVENT_TIME: time(),
"a": 1
})
@ -361,6 +364,7 @@ class MeowTests(unittest.TestCase):
EVENT_TYPE: "test",
EVENT_PATH: "path",
EVENT_RULE: rule,
EVENT_TIME: time(),
WATCHDOG_HASH: "hash",
WATCHDOG_BASE: "base"
})
@ -369,7 +373,8 @@ class MeowTests(unittest.TestCase):
valid_watchdog_event({
EVENT_TYPE: "test",
EVENT_PATH: "path",
EVENT_RULE: "rule"
EVENT_RULE: "rule",
EVENT_TIME: time()
})
with self.assertRaises(KeyError):
@ -377,6 +382,7 @@ class MeowTests(unittest.TestCase):
EVENT_TYPE: "anything",
EVENT_PATH: "path",
EVENT_RULE: "rule",
EVENT_TIME: time(),
"a": 1
})