NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added rudimentary conductor for job execution

Browse Source
This commit is contained in:
PatchOfScotland
2023-01-26 13:47:17 +01:00
parent 75de8147be
commit 31d06af5bf
18 changed files with 1895 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
tests/test_conductors.py Normal file
View File

@ -0,0 +1,244 @@
import os
import unittest
from core.correctness.vars import PYTHON_TYPE, TEST_HANDLER_BASE, SHA256, \
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, WATCHDOG_TYPE, \
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
from core.functionality import make_dir, rmtree, get_file_hash, create_event, \
create_job
from core.meow import create_rule
from conductors import LocalPythonConductor
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func
def failing_func():
raise Exception("bad function")
class MeowTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
make_dir(TEST_MONITOR_BASE)
make_dir(TEST_HANDLER_BASE)
make_dir(TEST_JOB_OUTPUT)
def tearDown(self)->None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree(TEST_HANDLER_BASE)
rmtree(TEST_JOB_OUTPUT)
def testLocalPythonConductorCreation(self)->None:
lpc = LocalPythonConductor()
valid_jobs = lpc.valid_job_types()
self.assertEqual(valid_jobs, [PYTHON_TYPE])
def testLocalPythonConductorValidJob(self)->None:
lpc = LocalPythonConductor()
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
with open(file_path, "w") as f:
f.write("Data")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":result_path
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
job_dict = create_job(
PYTHON_TYPE,
create_event(
WATCHDOG_TYPE,
file_path,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule,
WATCHDOG_HASH: file_hash
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
lpc.execute(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(result_path))
def testLocalPythonConductorBadArgs(self)->None:
lpc = LocalPythonConductor()
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
with open(file_path, "w") as f:
f.write("Data")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":result_path
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
bad_job_dict = create_job(
PYTHON_TYPE,
create_event(
WATCHDOG_TYPE,
file_path,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule,
WATCHDOG_HASH: file_hash
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
}
)
with self.assertRaises(KeyError):
lpc.execute(bad_job_dict)
# Ensure execution can continue after one failed job
good_job_dict = create_job(
PYTHON_TYPE,
create_event(
WATCHDOG_TYPE,
file_path,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule,
WATCHDOG_HASH: file_hash
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
lpc.execute(good_job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, good_job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(result_path))
def testLocalPythonConductorBadFunc(self)->None:
lpc = LocalPythonConductor()
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
with open(file_path, "w") as f:
f.write("Data")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":result_path
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
job_dict = create_job(
PYTHON_TYPE,
create_event(
WATCHDOG_TYPE,
file_path,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule,
WATCHDOG_HASH: file_hash
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_HASH: file_hash,
PYTHON_FUNC:failing_func,
}
)
with self.assertRaises(Exception):
lpc.execute(job_dict)

300
tests/test_functionality.py
View File

@ -1,22 +1,35 @@
import json
import unittest
import os
from datetime import datetime
from multiprocessing import Pipe, Queue
from time import sleep
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, EVENT_PATH
SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, EVENT_PATH, \
WATCHDOG_TYPE, PYTHON_TYPE, WATCHDOG_BASE, WATCHDOG_HASH, WATCHDOG_RULE, \
JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, \
PYTHON_EXECUTION_BASE, APPENDING_NOTEBOOK, JOB_ID, JOB_EVENT, JOB_TYPE, \
JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
make_dir, parameterize_jupyter_notebook, create_event
make_dir, parameterize_jupyter_notebook, create_event, create_job, \
replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \
KEYWORD_PATH, KEYWORD_REL_PATH, KEYWORD_DIR, KEYWORD_REL_DIR, \
KEYWORD_FILENAME, KEYWORD_PREFIX, KEYWORD_BASE, KEYWORD_EXTENSION, \
KEYWORD_JOB
from core.meow import create_rule
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
def setUp(self)->None:
super().setUp()
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
def tearDown(self) -> None:
def tearDown(self)->None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
@ -236,3 +249,280 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual(event2[EVENT_PATH], "path2")
self.assertEqual(event2["a"], 1)
def testCreateJob(self)->None:
pattern = FileEventPattern(
"pattern",
"file_path",
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"result_path"
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
event = create_event(
WATCHDOG_TYPE,
"file_path",
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule,
WATCHDOG_HASH: "file_hash"
}
)
job_dict = create_job(
PYTHON_TYPE,
event,
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":"file_path",
"outfile":"result_path"
},
JOB_HASH: "file_hash",
PYTHON_FUNC:max,
PYTHON_OUTPUT_DIR:"output",
PYTHON_EXECUTION_BASE:"execution"
}
)
self.assertIsInstance(job_dict, dict)
self.assertIn(JOB_ID, job_dict)
self.assertIsInstance(job_dict[JOB_ID], str)
self.assertIn(JOB_EVENT, job_dict)
self.assertEqual(job_dict[JOB_EVENT], event)
self.assertIn(JOB_TYPE, job_dict)
self.assertEqual(job_dict[JOB_TYPE], PYTHON_TYPE)
self.assertIn(JOB_PATTERN, job_dict)
self.assertEqual(job_dict[JOB_PATTERN], pattern)
self.assertIn(JOB_RECIPE, job_dict)
self.assertEqual(job_dict[JOB_RECIPE], recipe)
self.assertIn(JOB_RULE, job_dict)
self.assertEqual(job_dict[JOB_RULE], rule.name)
self.assertIn(JOB_STATUS, job_dict)
self.assertEqual(job_dict[JOB_STATUS], STATUS_QUEUED)
self.assertIn(JOB_CREATE_TIME, job_dict)
self.assertIsInstance(job_dict[JOB_CREATE_TIME], datetime)
self.assertIn(JOB_REQUIREMENTS, job_dict)
self.assertEqual(job_dict[JOB_REQUIREMENTS], {})
def testReplaceKeywords(self)->None:
test_dict = {
"A": f"--{KEYWORD_PATH}--",
"B": f"--{KEYWORD_REL_PATH}--",
"C": f"--{KEYWORD_DIR}--",
"D": f"--{KEYWORD_REL_DIR}--",
"E": f"--{KEYWORD_FILENAME}--",
"F": f"--{KEYWORD_PREFIX}--",
"G": f"--{KEYWORD_BASE}--",
"H": f"--{KEYWORD_EXTENSION}--",
"I": f"--{KEYWORD_JOB}--",
"J": f"--{KEYWORD_PATH}-{KEYWORD_PATH}--",
"K": f"{KEYWORD_PATH}",
"L": f"--{KEYWORD_PATH}-{KEYWORD_REL_PATH}-{KEYWORD_DIR}-"
f"{KEYWORD_REL_DIR}-{KEYWORD_FILENAME}-{KEYWORD_PREFIX}-"
f"{KEYWORD_BASE}-{KEYWORD_EXTENSION}-{KEYWORD_JOB}--",
"M": "A",
"N": 1
}
print(test_dict["A"])
replaced = replace_keywords(
test_dict, "job_id", "base/src/dir/file.ext", "base/monitor/dir")
self.assertIsInstance(replaced, dict)
self.assertEqual(len(test_dict.keys()), len(replaced.keys()))
for k in test_dict.keys():
self.assertIn(k, replaced)
self.assertEqual(replaced["A"], "--base/src/dir/file.ext--")
self.assertEqual(replaced["B"], "--../../src/dir/file.ext--")
self.assertEqual(replaced["C"], "--base/src/dir--")
self.assertEqual(replaced["D"], "--../../src/dir--")
self.assertEqual(replaced["E"], "--file.ext--")
self.assertEqual(replaced["F"], "--file--")
self.assertEqual(replaced["G"], "--base/monitor/dir--")
self.assertEqual(replaced["H"], "--.ext--")
self.assertEqual(replaced["I"], "--job_id--")
self.assertEqual(replaced["J"],
"--base/src/dir/file.ext-base/src/dir/file.ext--")
self.assertEqual(replaced["K"], "base/src/dir/file.ext")
self.assertEqual(replaced["L"],
"--base/src/dir/file.ext-../../src/dir/file.ext-base/src/dir-"
"../../src/dir-file.ext-file-base/monitor/dir-.ext-job_id--")
self.assertEqual(replaced["M"], "A")
self.assertEqual(replaced["N"], 1)
def testWriteNotebook(self)->None:
notebook_path = os.path.join(TEST_MONITOR_BASE, "test_notebook.ipynb")
self.assertFalse(os.path.exists(notebook_path))
write_notebook(APPENDING_NOTEBOOK, notebook_path)
self.assertTrue(os.path.exists(notebook_path))
with open(notebook_path, 'r') as f:
data = f.readlines()
print(data)
expected_bytes = [
'{"cells": [{"cell_type": "code", "execution_count": null, '
'"metadata": {}, "outputs": [], "source": ["# Default parameters '
'values\\n", "# The line to append\\n", "extra = \'This line '
'comes from a default pattern\'\\n", "# Data input file '
'location\\n", "infile = \'start/alpha.txt\'\\n", "# Output file '
'location\\n", "outfile = \'first/alpha.txt\'"]}, {"cell_type": '
'"code", "execution_count": null, "metadata": {}, "outputs": [], '
'"source": ["# load in dataset. This should be a text file\\n", '
'"with open(infile) as input_file:\\n", " data = '
'input_file.read()"]}, {"cell_type": "code", "execution_count": '
'null, "metadata": {}, "outputs": [], "source": ["# Append the '
'line\\n", "appended = data + \'\\\\n\' + extra"]}, {"cell_type": '
'"code", "execution_count": null, "metadata": {}, "outputs": [], '
'"source": ["import os\\n", "\\n", "# Create output directory if '
'it doesn\'t exist\\n", "output_dir_path = '
'os.path.dirname(outfile)\\n", "\\n", "if output_dir_path:\\n", '
'" os.makedirs(output_dir_path, exist_ok=True)\\n", "\\n", "# '
'Save added array as new dataset\\n", "with open(outfile, \'w\') '
'as output_file:\\n", " output_file.write(appended)"]}], '
'"metadata": {"kernelspec": {"display_name": "Python 3", '
'"language": "python", "name": "python3"}, "language_info": '
'{"codemirror_mode": {"name": "ipython", "version": 3}, '
'"file_extension": ".py", "mimetype": "text/x-python", "name": '
'"python", "nbconvert_exporter": "python", "pygments_lexer": '
'"ipython3", "version": "3.10.6 (main, Nov 14 2022, 16:10:14) '
'[GCC 11.3.0]"}, "vscode": {"interpreter": {"hash": '
'"916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1'
'"}}}, "nbformat": 4, "nbformat_minor": 4}'
]
self.assertEqual(data, expected_bytes)
def testReadNotebook(self)->None:
notebook_path = os.path.join(TEST_MONITOR_BASE, "test_notebook.ipynb")
write_notebook(APPENDING_NOTEBOOK, notebook_path)
notebook = read_notebook(notebook_path)
self.assertEqual(notebook, APPENDING_NOTEBOOK)
with self.assertRaises(FileNotFoundError):
read_notebook("doesNotExist.ipynb")
filepath = os.path.join(TEST_MONITOR_BASE, "T.txt")
with open(filepath, "w") as f:
f.write("Data")
with self.assertRaises(ValueError):
read_notebook(filepath)
filepath = os.path.join(TEST_MONITOR_BASE, "T.ipynb")
with open(filepath, "w") as f:
f.write("Data")
with self.assertRaises(json.decoder.JSONDecodeError):
read_notebook(filepath)
def testWriteYaml(self)->None:
yaml_dict = {
"A": "a",
"B": 1,
"C": {
"D": True,
"E": [
1, 2, 3
]
}
}
filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
self.assertFalse(os.path.exists(filepath))
write_yaml(yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
with open(filepath, 'r') as f:
data = f.readlines()
expected_bytes = [
'A: a\n',
'B: 1\n',
'C:\n',
' D: true\n',
' E:\n',
' - 1\n',
' - 2\n',
' - 3\n'
]
self.assertEqual(data, expected_bytes)
def testReadYaml(self)->None:
yaml_dict = {
"A": "a",
"B": 1,
"C": {
"D": True,
"E": [
1, 2, 3
]
}
}
filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
write_yaml(yaml_dict, filepath)
read_dict = read_yaml(filepath)
self.assertEqual(yaml_dict, read_dict)
with self.assertRaises(FileNotFoundError):
read_yaml("doesNotExist")
filepath = os.path.join(TEST_MONITOR_BASE, "T.txt")
with open(filepath, "w") as f:
f.write("Data")
data = read_yaml(filepath)
self.assertEqual(data, "Data")
def testMakeDir(self)->None:
testDir = os.path.join(TEST_MONITOR_BASE, "Test")
self.assertFalse(os.path.exists(testDir))
make_dir(testDir)
self.assertTrue(os.path.exists(testDir))
self.assertTrue(os.path.isdir(testDir))
nested = os.path.join(TEST_MONITOR_BASE, "A", "B", "C", "D")
self.assertFalse(os.path.exists(os.path.join(TEST_MONITOR_BASE, "A")))
make_dir(nested)
self.assertTrue(os.path.exists(nested))
with self.assertRaises(FileExistsError):
make_dir(nested, can_exist=False)
filepath = os.path.join(TEST_MONITOR_BASE, "T.txt")
with open(filepath, "w") as f:
f.write("Data")
with self.assertRaises(ValueError):
make_dir(filepath)
halfway = os.path.join(TEST_MONITOR_BASE, "A", "B")
make_dir(halfway, ensure_clean=True)
self.assertTrue(os.path.exists(halfway))
self.assertEqual(len(os.listdir(halfway)), 0)
def testRemoveTree(self)->None:
nested = os.path.join(TEST_MONITOR_BASE, "A", "B")
self.assertFalse(os.path.exists(os.path.join(TEST_MONITOR_BASE, "A")))
make_dir(nested)
self.assertTrue(os.path.exists(nested))
rmtree(os.path.join(TEST_MONITOR_BASE, "A"))
self.assertTrue(os.path.exists(TEST_MONITOR_BASE))
self.assertFalse(os.path.exists(os.path.join(TEST_MONITOR_BASE, "A")))
self.assertFalse(os.path.exists(
os.path.join(TEST_MONITOR_BASE, "A", "B")))

173
tests/test_meow.py
View File

@ -1,18 +1,14 @@
import io
import os
import unittest
from multiprocessing import Pipe
from typing import Any, Union
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
TEST_MONITOR_BASE, BAREBONES_NOTEBOOK, WATCHDOG_BASE, WATCHDOG_RULE, \
EVENT_PATH, WATCHDOG_TYPE, EVENT_TYPE
TEST_MONITOR_BASE, BAREBONES_NOTEBOOK
from core.functionality import make_dir, rmtree
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, create_rules
from patterns import FileEventPattern, WatchdogMonitor
BaseHandler, BaseConductor, create_rules, create_rule
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
valid_pattern_one = FileEventPattern(
@ -27,13 +23,13 @@ valid_recipe_two = JupyterNotebookRecipe(
class MeowTests(unittest.TestCase):
def setUp(self) -> None:
def setUp(self)->None:
super().setUp()
make_dir(TEST_MONITOR_BASE)
make_dir(TEST_HANDLER_BASE)
make_dir(TEST_JOB_OUTPUT)
def tearDown(self) -> None:
def tearDown(self)->None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree(TEST_HANDLER_BASE)
@ -93,8 +89,18 @@ class MeowTests(unittest.TestCase):
pass
FullRule("name", "", "")
def testCreateRule(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
self.assertIsInstance(rule, BaseRule)
with self.assertRaises(ValueError):
rule = create_rule(valid_pattern_one, valid_recipe_two)
def testCreateRulesMinimum(self)->None:
create_rules({}, {})
rules = create_rules({}, {})
self.assertEqual(len(rules), 0)
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
@ -166,135 +172,9 @@ class MeowTests(unittest.TestCase):
pass
def get_rules(self)->None:
pass
FullTestMonitor({}, {})
def testMonitoring(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
monitor_debug_stream = io.StringIO("")
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
patterns,
recipes,
print=monitor_debug_stream,
logging=3,
settletime=1
)
rules = wm.get_rules()
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.start()
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
messages = []
while True:
if from_monitor_reader.poll(3):
messages.append(from_monitor_reader.recv())
else:
break
self.assertTrue(len(messages), 1)
message = messages[0]
self.assertEqual(type(message), dict)
self.assertIn(EVENT_TYPE, message)
self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE)
self.assertIn(WATCHDOG_BASE, message)
self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE)
self.assertIn(EVENT_PATH, message)
self.assertEqual(message[EVENT_PATH],
os.path.join(start_dir, "A.txt"))
self.assertIn(WATCHDOG_RULE, message)
self.assertEqual(message[WATCHDOG_RULE].name, rule.name)
wm.stop()
def testMonitoringRetroActive(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
monitor_debug_stream = io.StringIO("")
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
patterns,
recipes,
print=monitor_debug_stream,
logging=3,
settletime=1
)
rules = wm.get_rules()
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.start()
messages = []
while True:
if from_monitor_reader.poll(3):
messages.append(from_monitor_reader.recv())
else:
break
self.assertTrue(len(messages), 1)
message = messages[0]
self.assertEqual(type(message), dict)
self.assertIn(EVENT_TYPE, message)
self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE)
self.assertIn(WATCHDOG_BASE, message)
self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE)
self.assertIn(EVENT_PATH, message)
self.assertEqual(message[EVENT_PATH],
os.path.join(start_dir, "A.txt"))
self.assertIn(WATCHDOG_RULE, message)
self.assertEqual(message[WATCHDOG_RULE].name, rule.name)
wm.stop()
def testBaseHandler(self)->None:
with self.assertRaises(TypeError):
BaseHandler()
@ -316,5 +196,24 @@ class MeowTests(unittest.TestCase):
pass
def valid_event_types(self)->list[str]:
pass
FullTestHandler()
def testBaseConductor(self)->None:
with self.assertRaises(NotImplementedError):
BaseConductor()
class TestConductor(BaseConductor):
pass
with self.assertRaises(NotImplementedError):
TestConductor()
class FullTestConductor(BaseConductor):
def execute(self, job:dict[str,Any])->None:
pass
def valid_job_types(self)->list[str]:
pass
FullTestConductor()

518
tests/test_patterns.py
View File

@ -1,4 +1,5 @@
import io
import os
import unittest
@ -8,17 +9,38 @@ from core.correctness.vars import FILE_CREATE_EVENT, BAREBONES_NOTEBOOK, \
TEST_MONITOR_BASE, EVENT_TYPE, WATCHDOG_RULE, WATCHDOG_BASE, \
WATCHDOG_TYPE, EVENT_PATH
from core.functionality import rmtree, make_dir
from core.meow import create_rules
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
_DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP
from recipes import JupyterNotebookRecipe
def patterns_equal(tester, pattern_one, pattern_two):
tester.assertEqual(pattern_one.name, pattern_two.name)
tester.assertEqual(pattern_one.recipe, pattern_two.recipe)
tester.assertEqual(pattern_one.parameters, pattern_two.parameters)
tester.assertEqual(pattern_one.outputs, pattern_two.outputs)
tester.assertEqual(pattern_one.triggering_path,
pattern_two.triggering_path)
tester.assertEqual(pattern_one.triggering_file,
pattern_two.triggering_file)
tester.assertEqual(pattern_one.event_mask, pattern_two.event_mask)
tester.assertEqual(pattern_one.sweep, pattern_two.sweep)
def recipes_equal(tester, recipe_one, recipe_two):
tester.assertEqual(recipe_one.name, recipe_two.name)
tester.assertEqual(recipe_one.recipe, recipe_two.recipe)
tester.assertEqual(recipe_one.parameters, recipe_two.parameters)
tester.assertEqual(recipe_one.requirements, recipe_two.requirements)
tester.assertEqual(recipe_one.source, recipe_two.source)
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
def setUp(self)->None:
super().setUp()
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
def tearDown(self) -> None:
def tearDown(self)->None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
@ -199,3 +221,493 @@ class CorrectnessTests(unittest.TestCase):
self.assertIsNone(new_message)
wm.stop()
def testMonitoring(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
monitor_debug_stream = io.StringIO("")
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
patterns,
recipes,
print=monitor_debug_stream,
logging=3,
settletime=1
)
rules = wm.get_rules()
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.start()
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
messages = []
while True:
if from_monitor_reader.poll(3):
messages.append(from_monitor_reader.recv())
else:
break
self.assertTrue(len(messages), 1)
message = messages[0]
self.assertEqual(type(message), dict)
self.assertIn(EVENT_TYPE, message)
self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE)
self.assertIn(WATCHDOG_BASE, message)
self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE)
self.assertIn(EVENT_PATH, message)
self.assertEqual(message[EVENT_PATH],
os.path.join(start_dir, "A.txt"))
self.assertIn(WATCHDOG_RULE, message)
self.assertEqual(message[WATCHDOG_RULE].name, rule.name)
wm.stop()
def testMonitoringRetroActive(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
monitor_debug_stream = io.StringIO("")
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
patterns,
recipes,
print=monitor_debug_stream,
logging=3,
settletime=1
)
rules = wm.get_rules()
rule = rules[list(rules.keys())[0]]
from_monitor_reader, from_monitor_writer = Pipe()
wm.to_runner = from_monitor_writer
wm.start()
messages = []
while True:
if from_monitor_reader.poll(3):
messages.append(from_monitor_reader.recv())
else:
break
self.assertTrue(len(messages), 1)
message = messages[0]
self.assertEqual(type(message), dict)
self.assertIn(EVENT_TYPE, message)
self.assertEqual(message[EVENT_TYPE], WATCHDOG_TYPE)
self.assertIn(WATCHDOG_BASE, message)
self.assertEqual(message[WATCHDOG_BASE], TEST_MONITOR_BASE)
self.assertIn(EVENT_PATH, message)
self.assertEqual(message[EVENT_PATH],
os.path.join(start_dir, "A.txt"))
self.assertIn(WATCHDOG_RULE, message)
self.assertEqual(message[WATCHDOG_RULE].name, rule.name)
wm.stop()
def testMonitorGetPatterns(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
parameters={})
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{
pattern_one.name: pattern_one,
pattern_two.name: pattern_two
},
{}
)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 2)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
self.assertIn(pattern_two.name, patterns)
patterns_equal(self, patterns[pattern_two.name], pattern_two)
def testMonitorAddPattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
parameters={})
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{pattern_one.name: pattern_one},
{}
)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 1)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
wm.add_pattern(pattern_two)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 2)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
self.assertIn(pattern_two.name, patterns)
patterns_equal(self, patterns[pattern_two.name], pattern_two)
with self.assertRaises(KeyError):
wm.add_pattern(pattern_two)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 2)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
self.assertIn(pattern_two.name, patterns)
patterns_equal(self, patterns[pattern_two.name], pattern_two)
def testMonitorUpdatePattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
parameters={})
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{pattern_one.name: pattern_one},
{}
)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 1)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
pattern_one.recipe = "top_secret_recipe"
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 1)
self.assertIn(pattern_one.name, patterns)
self.assertEqual(patterns[pattern_one.name].name,
pattern_one.name)
self.assertEqual(patterns[pattern_one.name].recipe,
"recipe_one")
self.assertEqual(patterns[pattern_one.name].parameters,
pattern_one.parameters)
self.assertEqual(patterns[pattern_one.name].outputs,
pattern_one.outputs)
self.assertEqual(patterns[pattern_one.name].triggering_path,
pattern_one.triggering_path)
self.assertEqual(patterns[pattern_one.name].triggering_file,
pattern_one.triggering_file)
self.assertEqual(patterns[pattern_one.name].event_mask,
pattern_one.event_mask)
self.assertEqual(patterns[pattern_one.name].sweep,
pattern_one.sweep)
wm.update_pattern(pattern_one)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 1)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
with self.assertRaises(KeyError):
wm.update_pattern(pattern_two)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 1)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
def testMonitorRemovePattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
parameters={})
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{pattern_one.name: pattern_one},
{}
)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 1)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
with self.assertRaises(KeyError):
wm.remove_pattern(pattern_two)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 1)
self.assertIn(pattern_one.name, patterns)
patterns_equal(self, patterns[pattern_one.name], pattern_one)
wm.remove_pattern(pattern_one)
patterns = wm.get_patterns()
self.assertIsInstance(patterns, dict)
self.assertEqual(len(patterns), 0)
def testMonitorGetRecipes(self)->None:
recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{},
{
recipe_one.name: recipe_one,
recipe_two.name: recipe_two
}
)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 2)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
self.assertIn(recipe_two.name, recipes)
recipes_equal(self, recipes[recipe_two.name], recipe_two)
def testMonitorAddRecipe(self)->None:
recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{},
{
recipe_one.name: recipe_one
}
)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 1)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
wm.add_recipe(recipe_two)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 2)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
self.assertIn(recipe_two.name, recipes)
recipes_equal(self, recipes[recipe_two.name], recipe_two)
with self.assertRaises(KeyError):
wm.add_recipe(recipe_two)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 2)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
self.assertIn(recipe_two.name, recipes)
recipes_equal(self, recipes[recipe_two.name], recipe_two)
def testMonitorUpdateRecipe(self)->None:
recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{},
{
recipe_one.name: recipe_one
}
)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 1)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
recipe_one.source = "top_secret_source"
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 1)
self.assertIn(recipe_one.name, recipes)
self.assertEqual(recipes[recipe_one.name].name,
recipe_one.name)
self.assertEqual(recipes[recipe_one.name].recipe,
recipe_one.recipe)
self.assertEqual(recipes[recipe_one.name].parameters,
recipe_one.parameters)
self.assertEqual(recipes[recipe_one.name].requirements,
recipe_one.requirements)
self.assertEqual(recipes[recipe_one.name].source,
"")
wm.update_recipe(recipe_one)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 1)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
with self.assertRaises(KeyError):
wm.update_recipe(recipe_two)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 1)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
def testMonitorRemoveRecipe(self)->None:
recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
{},
{
recipe_one.name: recipe_one
}
)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 1)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
with self.assertRaises(KeyError):
wm.remove_recipe(recipe_two)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 1)
self.assertIn(recipe_one.name, recipes)
recipes_equal(self, recipes[recipe_one.name], recipe_one)
wm.remove_recipe(recipe_one)
recipes = wm.get_recipes()
self.assertIsInstance(recipes, dict)
self.assertEqual(len(recipes), 0)
def testMonitorGetRules(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
parameters={})
recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
pattern_two.name: pattern_two,
}
recipes = {
recipe_one.name: recipe_one,
recipe_two.name: recipe_two,
}
wm = WatchdogMonitor(
TEST_MONITOR_BASE,
patterns,
recipes
)
rules = wm.get_rules()
self.assertIsInstance(rules, dict)
self.assertEqual(len(rules), 2)

138
tests/test_recipes.py
View File

@ -1,29 +1,34 @@
import io
import jsonschema
import os
import unittest
from time import sleep
from multiprocessing import Pipe
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, BASE_FILE, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from core.correctness.vars import BAREBONES_NOTEBOOK, TEST_HANDLER_BASE, \
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, \
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_TYPE, EVENT_PATH
from core.functionality import rmtree, make_dir, read_notebook
from core.meow import create_rules
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_TYPE, EVENT_PATH, SHA256, \
WATCHDOG_HASH, JOB_ID, PYTHON_TYPE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, \
APPENDING_NOTEBOOK, META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, \
RESULT_FILE
from core.correctness.validation import valid_job
from core.functionality import rmtree, make_dir, get_file_hash, create_job, \
create_event
from core.meow import create_rules, create_rule
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, job_func
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
def setUp(self)->None:
super().setUp()
make_dir(TEST_MONITOR_BASE)
make_dir(TEST_HANDLER_BASE)
make_dir(TEST_JOB_OUTPUT)
def tearDown(self) -> None:
def tearDown(self)->None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree(TEST_HANDLER_BASE)
@ -98,14 +103,12 @@ class CorrectnessTests(unittest.TestCase):
)
def testPapermillHandlerHandling(self)->None:
debug_stream = io.StringIO("")
from_handler_reader, from_handler_writer = Pipe()
ph = PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=debug_stream,
logging=3
TEST_JOB_OUTPUT
)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
@ -133,41 +136,88 @@ class CorrectnessTests(unittest.TestCase):
EVENT_TYPE: WATCHDOG_TYPE,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule
WATCHDOG_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
loops = 0
job_id = None
while loops < 15:
sleep(1)
debug_stream.seek(0)
messages = debug_stream.readlines()
if from_handler_reader.poll(3):
job = from_handler_reader.recv()
for msg in messages:
self.assertNotIn("ERROR", msg)
if "INFO: Completed job " in msg:
job_id = msg.replace("INFO: Completed job ", "")
job_id = job_id[:job_id.index(" with output")]
loops = 15
loops += 1
self.assertIsNotNone(job[JOB_ID])
self.assertIsNotNone(job_id)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
valid_job(job)
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(job_dir)), 5)
def testJobFunc(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
self.assertIn(META_FILE, os.listdir(job_dir))
self.assertIn(BASE_FILE, os.listdir(job_dir))
self.assertIn(PARAMS_FILE, os.listdir(job_dir))
self.assertIn(JOB_FILE, os.listdir(job_dir))
self.assertIn(RESULT_FILE, os.listdir(job_dir))
with open(file_path, "w") as f:
f.write("Data")
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
file_hash = get_file_hash(file_path, SHA256)
self.assertEqual("124875.0\n",
result["cells"][4]["outputs"][0]["text"][0])
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":result_path
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
job_dict = create_job(
PYTHON_TYPE,
create_event(
WATCHDOG_TYPE,
file_path,
{
WATCHDOG_BASE: TEST_MONITOR_BASE,
WATCHDOG_RULE: rule,
WATCHDOG_HASH: file_hash
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
job_func(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(result_path))
def testJobFuncBadArgs(self)->None:
try:
job_func({})
except Exception:
pass
self.assertEqual(len(os.listdir(TEST_HANDLER_BASE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)

4
tests/test_rules.py
View File

@ -7,10 +7,10 @@ from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
def setUp(self)->None:
return super().setUp()
def tearDown(self) -> None:
def tearDown(self)->None:
return super().tearDown()
def testFileEventJupyterNotebookRuleCreationMinimum(self)->None:

207
tests/test_runner.py
View File

@ -5,30 +5,171 @@ import unittest
from time import sleep
from conductors import LocalPythonConductor
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
TEST_MONITOR_BASE, APPENDING_NOTEBOOK
TEST_MONITOR_BASE, APPENDING_NOTEBOOK, RESULT_FILE
from core.functionality import make_dir, rmtree, read_notebook
from core.meow import create_rules
from core.meow import BaseMonitor, BaseHandler, BaseConductor
from core.runner import MeowRunner
from patterns import WatchdogMonitor, FileEventPattern
from recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe, RESULT_FILE
JupyterNotebookRecipe
class MeowTests(unittest.TestCase):
def setUp(self) -> None:
def setUp(self)->None:
super().setUp()
make_dir(TEST_MONITOR_BASE)
make_dir(TEST_HANDLER_BASE)
make_dir(TEST_JOB_OUTPUT)
def tearDown(self) -> None:
def tearDown(self)->None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree(TEST_HANDLER_BASE)
rmtree(TEST_JOB_OUTPUT)
def testMeowRunner(self)->None:
def testMeowRunnerSetup(self)->None:
monitor_one = WatchdogMonitor(TEST_MONITOR_BASE, {}, {})
monitor_two = WatchdogMonitor(TEST_MONITOR_BASE, {}, {})
monitors = [ monitor_one, monitor_two ]
handler_one = PapermillHandler(TEST_HANDLER_BASE, TEST_JOB_OUTPUT)
handler_two = PapermillHandler(TEST_HANDLER_BASE, TEST_JOB_OUTPUT)
handlers = [ handler_one, handler_two ]
conductor_one = LocalPythonConductor()
conductor_two = LocalPythonConductor()
conductors = [ conductor_one, conductor_two ]
runner = MeowRunner(monitor_one, handler_one, conductor_one)
self.assertIsInstance(runner.monitors, list)
for m in runner.monitors:
self.assertIsInstance(m, BaseMonitor)
self.assertEqual(len(runner.monitors), 1)
self.assertEqual(runner.monitors[0], monitor_one)
self.assertIsInstance(runner.from_monitors, list)
self.assertEqual(len(runner.from_monitors), 1)
runner.monitors[0].to_runner.send("monitor test message")
message = None
if runner.from_monitors[0].poll(3):
message = runner.from_monitors[0].recv()
self.assertIsNotNone(message)
self.assertEqual(message, "monitor test message")
self.assertIsInstance(runner.handlers, dict)
for handler_list in runner.handlers.values():
for h in handler_list:
self.assertIsInstance(h, BaseHandler)
self.assertEqual(
len(runner.handlers.keys()), len(handler_one.valid_event_types()))
for event_type in handler_one.valid_event_types():
self.assertIn(event_type, runner.handlers.keys())
self.assertEqual(len(runner.handlers[event_type]), 1)
self.assertEqual(runner.handlers[event_type][0], handler_one)
self.assertIsInstance(runner.from_handlers, list)
self.assertEqual(len(runner.from_handlers), 1)
runner.handlers[handler_one.valid_event_types()[0]][0].to_runner.send(
"handler test message")
message = None
if runner.from_handlers[0].poll(3):
message = runner.from_handlers[0].recv()
self.assertIsNotNone(message)
self.assertEqual(message, "handler test message")
self.assertIsInstance(runner.conductors, dict)
for conductor_list in runner.conductors.values():
for c in conductor_list:
self.assertIsInstance(c, BaseConductor)
self.assertEqual(
len(runner.conductors.keys()), len(conductor_one.valid_job_types()))
for job_type in conductor_one.valid_job_types():
self.assertIn(job_type, runner.conductors.keys())
self.assertEqual(len(runner.conductors[job_type]), 1)
self.assertEqual(runner.conductors[job_type][0], conductor_one)
runner = MeowRunner(monitors, handlers, conductors)
self.assertIsInstance(runner.monitors, list)
for m in runner.monitors:
self.assertIsInstance(m, BaseMonitor)
self.assertEqual(len(runner.monitors), len(monitors))
self.assertIn(monitor_one, runner.monitors)
self.assertIn(monitor_two, runner.monitors)
self.assertIsInstance(runner.from_monitors, list)
self.assertEqual(len(runner.from_monitors), len(monitors))
for rm in runner.monitors:
rm.to_runner.send("monitor test message")
messages = [None] * len(monitors)
for i, rfm in enumerate(runner.from_monitors):
if rfm.poll(3):
messages[i] = rfm.recv()
for m in messages:
self.assertIsNotNone(m)
self.assertEqual(m, "monitor test message")
self.assertIsInstance(runner.handlers, dict)
for handler_list in runner.handlers.values():
for h in handler_list:
self.assertIsInstance(h, BaseHandler)
all_events = []
for h in handlers:
for e in h.valid_event_types():
if e not in all_events:
all_events.append(e)
self.assertEqual(len(runner.handlers.keys()), len(all_events))
for handler in handlers:
for event_type in handler.valid_event_types():
relevent_handlers = [h for h in handlers
if event_type in h.valid_event_types()]
self.assertIn(event_type, runner.handlers.keys())
self.assertEqual(len(runner.handlers[event_type]),
len(relevent_handlers))
for rh in relevent_handlers:
self.assertIn(rh, runner.handlers[event_type])
self.assertIsInstance(runner.from_handlers, list)
self.assertEqual(len(runner.from_handlers), len(handlers))
runner_handlers = []
for handler_list in runner.handlers.values():
for h in handler_list:
runner_handlers.append(h)
runner_handlers = [h for h in handler_list for
handler_list in runner.handlers.values()]
for rh in handler_list:
rh.to_runner.send("handler test message")
message = None
if runner.from_handlers[0].poll(3):
message = runner.from_handlers[0].recv()
self.assertIsNotNone(message)
self.assertEqual(message, "handler test message")
self.assertIsInstance(runner.conductors, dict)
for conductor_list in runner.conductors.values():
for c in conductor_list:
self.assertIsInstance(c, BaseConductor)
all_jobs = []
for c in conductors:
for j in c.valid_job_types():
if j not in all_jobs:
all_jobs.append(j)
self.assertEqual(len(runner.conductors.keys()), len(all_jobs))
for conductor in conductors:
for job_type in conductor.valid_job_types():
relevent_conductors = [c for c in conductors
if job_type in c.valid_job_types()]
self.assertIn(job_type, runner.conductors.keys())
self.assertEqual(len(runner.conductors[job_type]),
len(relevent_conductors))
for rc in relevent_conductors:
self.assertIn(rc, runner.conductors[job_type])
def testMeowRunnerExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={
@ -45,24 +186,22 @@ class MeowTests(unittest.TestCase):
recipe.name: recipe,
}
monitor_debug_stream = io.StringIO("")
handler_debug_stream = io.StringIO("")
runner_debug_stream = io.StringIO("")
runner = MeowRunner(
WatchdogMonitor(
TEST_MONITOR_BASE,
patterns,
recipes,
print=monitor_debug_stream,
logging=3,
settletime=1
),
PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=handler_debug_stream,
logging=3
)
),
LocalPythonConductor(),
print=runner_debug_stream,
logging=3
)
runner.start()
@ -79,18 +218,22 @@ class MeowTests(unittest.TestCase):
job_id = None
while loops < 15:
sleep(1)
handler_debug_stream.seek(0)
messages = handler_debug_stream.readlines()
runner_debug_stream.seek(0)
messages = runner_debug_stream.readlines()
for msg in messages:
self.assertNotIn("ERROR", msg)
if "INFO: Completed job " in msg:
job_id = msg.replace("INFO: Completed job ", "")
job_id = job_id[:job_id.index(" with output")]
if "INFO: Completed execution for job: '" in msg:
job_id = msg.replace(
"INFO: Completed execution for job: '", "")
job_id = job_id[:-2]
loops = 15
loops += 1
print("JOB ID:")
print(job_id)
self.assertIsNotNone(job_id)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
@ -111,7 +254,7 @@ class MeowTests(unittest.TestCase):
self.assertEqual(data, "Initial Data\nA line from a test Pattern")
def testMeowRunnerLinkeExecution(self)->None:
def testMeowRunnerLinkedExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={
@ -134,26 +277,23 @@ class MeowTests(unittest.TestCase):
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
monitor_debug_stream = io.StringIO("")
handler_debug_stream = io.StringIO("")
runner_debug_stream = io.StringIO("")
runner = MeowRunner(
WatchdogMonitor(
TEST_MONITOR_BASE,
patterns,
recipes,
print=monitor_debug_stream,
logging=3,
settletime=1
),
PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=handler_debug_stream,
logging=3
)
TEST_JOB_OUTPUT
),
LocalPythonConductor(),
print=runner_debug_stream,
logging=3
)
runner.start()
@ -170,15 +310,16 @@ class MeowTests(unittest.TestCase):
job_ids = []
while len(job_ids) < 2 and loops < 15:
sleep(1)
handler_debug_stream.seek(0)
messages = handler_debug_stream.readlines()
runner_debug_stream.seek(0)
messages = runner_debug_stream.readlines()
for msg in messages:
self.assertNotIn("ERROR", msg)
if "INFO: Completed job " in msg:
job_id = msg.replace("INFO: Completed job ", "")
job_id = job_id[:job_id.index(" with output")]
if "INFO: Completed execution for job: '" in msg:
job_id = msg.replace(
"INFO: Completed execution for job: '", "")
job_id = job_id[:-2]
if job_id not in job_ids:
job_ids.append(job_id)
loops += 1

47
tests/test_validation.py
View File

@ -1,22 +1,26 @@
import io
import unittest
import os
from datetime import datetime
from typing import Any, Union
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_existing_dir_path, valid_non_existing_path, valid_event
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
setup_debugging
from core.correctness.vars import VALID_NAME_CHARS, TEST_MONITOR_BASE, \
SHA256, EVENT_TYPE, EVENT_PATH
SHA256, EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME
from core.functionality import rmtree, make_dir
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
def setUp(self)->None:
super().setUp()
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
def tearDown(self) -> None:
def tearDown(self)->None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree("first")
@ -219,3 +223,38 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(KeyError):
valid_event({})
def testJobValidation(self)->None:
valid_job({
JOB_TYPE: "test",
JOB_EVENT: {},
JOB_ID: "id",
JOB_PATTERN: "pattern",
JOB_RECIPE: "recipe",
JOB_RULE: "rule",
JOB_STATUS: "status",
JOB_CREATE_TIME: datetime.now()
})
with self.assertRaises(KeyError):
valid_job({JOB_TYPE: "test"})
with self.assertRaises(KeyError):
valid_job({"JOB_TYPE": "test"})
with self.assertRaises(KeyError):
valid_job({})
def testSetupDebugging(self)->None:
stream = io.StringIO("")
target, level = setup_debugging(stream, 1)
self.assertIsInstance(target, io.StringIO)
self.assertIsInstance(level, int)
with self.assertRaises(TypeError):
setup_debugging("stream", 1)
with self.assertRaises(TypeError):
setup_debugging(stream, "1")