NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

differentiated papermill and python jobs more clearly

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-03 14:47:16 +01:00
parent 72d6b263b7
commit 47f9fe73fa
17 changed files with 853 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tests/shared.py
View File

@ -22,6 +22,31 @@ def teardown():
rmtree(TEST_JOB_OUTPUT)
rmtree("first")
# Recipe funcs
BAREBONES_PYTHON_SCRIPT = [
""
]
COMPLETE_PYTHON_SCRIPT = [
"# Setup parameters",
"num = 1000",
"infile = 'somehere/particular'",
"outfile = 'nowhere/particular'",
"",
"with open(infile, 'r') as file:",
" s = int(file.read())",
""
"for i in range(num):",
" s += i",
"",
"div_by = 4",
"result = s / div_by",
"",
"print(result)",
"",
"with open(outfile, 'w') as file:",
" file.write(str(result))"
]
# Jupyter notebooks
BAREBONES_NOTEBOOK = {
"cells": [],
@ -261,4 +286,4 @@ ADDING_NOTEBOOK = {
},
"nbformat": 4,
"nbformat_minor": 4
}
}

154
tests/test_conductors.py
View File

@ -4,15 +4,20 @@ import unittest
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \
META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
STATUS_DONE, JOB_TYPE_PAPERMILL, get_base_file, get_result_file, \
get_job_file
from core.functionality import get_file_hash, create_watchdog_event, \
create_job, make_dir, write_yaml, write_notebook
create_job, make_dir, write_yaml, write_notebook, read_yaml, write_file, \
lines_to_string
from core.meow import create_rule
from conductors import LocalPythonConductor
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from recipes.python_recipe import PythonRecipe, python_job_func
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
TEST_JOB_OUTPUT, TEST_HANDLER_BASE
TEST_JOB_OUTPUT, TEST_HANDLER_BASE, COMPLETE_PYTHON_SCRIPT
def failing_func():
@ -32,10 +37,94 @@ class MeowTests(unittest.TestCase):
def testLocalPythonConductorCreation(self)->None:
LocalPythonConductor()
#TODO Test LocalPythonConductor execution criteria
#TODO Test LocalPythonConductor executes valid python jobs
def testLocalPythonConductorValidPythonJob(self)->None:
lpc = LocalPythonConductor()
# Test LocalPythonConductor executes valid jobs
def testLocalPythonConductorValidJob(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
with open(file_path, "w") as f:
f.write("150")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"num":450,
"outfile":result_path
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
rule = create_rule(pattern, recipe)
params_dict = {
"num":450,
"infile":file_path,
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_PYTHON,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
lpc.execute(job_dict)
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor executes valid papermill jobs
def testLocalPythonConductorValidPapermillJob(self)->None:
lpc = LocalPythonConductor()
file_path = os.path.join(TEST_MONITOR_BASE, "test")
@ -67,7 +156,7 @@ class MeowTests(unittest.TestCase):
}
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -77,7 +166,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -89,7 +178,10 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(job_dict)
@ -99,11 +191,22 @@ class MeowTests(unittest.TestCase):
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
@ -140,7 +243,7 @@ class MeowTests(unittest.TestCase):
}
bad_job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -150,7 +253,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
}
)
@ -160,7 +263,7 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
with self.assertRaises(KeyError):
@ -168,7 +271,7 @@ class MeowTests(unittest.TestCase):
# Ensure execution can continue after one failed job
good_job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -178,7 +281,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -190,7 +293,7 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(good_job_dict)
@ -201,10 +304,13 @@ class MeowTests(unittest.TestCase):
output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
@ -235,7 +341,7 @@ class MeowTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,

13
tests/test_functionality.py
View File

@ -12,7 +12,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
make_dir, parameterize_jupyter_notebook, create_event, create_job, \
replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \
@ -240,6 +240,8 @@ class CorrectnessTests(unittest.TestCase):
pn["cells"][0]["source"],
"# The first cell\n\ns = 4\nnum = 1000")
# TODO Test that parameterize_python_script parameterises given script
# Test that create_event produces valid event dictionary
def testCreateEvent(self)->None:
pattern = FileEventPattern(
@ -307,7 +309,7 @@ class CorrectnessTests(unittest.TestCase):
)
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
event,
extras={
JOB_PARAMETERS:{
@ -328,7 +330,7 @@ class CorrectnessTests(unittest.TestCase):
self.assertIn(JOB_EVENT, job_dict)
self.assertEqual(job_dict[JOB_EVENT], event)
self.assertIn(JOB_TYPE, job_dict)
self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PYTHON)
self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PAPERMILL)
self.assertIn(JOB_PATTERN, job_dict)
self.assertEqual(job_dict[JOB_PATTERN], pattern.name)
self.assertIn(JOB_RECIPE, job_dict)
@ -659,3 +661,8 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event["a"], 1)
self.assertEqual(event[WATCHDOG_BASE], "base")
#TODO test read file
#TODO test readlines file
#TODO test write file
#TODO test lines to str

77
tests/test_meow.py
View File

@ -3,6 +3,7 @@ import unittest
from typing import Any, Union, Tuple
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, BaseConductor, create_rules, create_rule
from patterns import FileEventPattern
@ -70,6 +71,78 @@ class MeowTests(unittest.TestCase):
pass
FullPattern("name", "", "", "", "")
# Test expansion of parameter sweeps
def testBasePatternExpandSweeps(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={
"s1":{
SWEEP_START: 10, SWEEP_STOP: 20, SWEEP_JUMP:5
}
})
es = pattern_one.expand_sweeps()
self.assertIsInstance(es, list)
self.assertEqual(len(es), 3)
values = [
"s1-10", "s1-15", "s1-20",
]
for sweep_vals in es:
self.assertIsInstance(sweep_vals, tuple)
self.assertEqual(len(sweep_vals), 1)
val1 = None
for sweep_val in sweep_vals:
self.assertIsInstance(sweep_val, tuple)
self.assertEqual(len(sweep_val), 2)
if sweep_val[0] == "s1":
val1 = f"s1-{sweep_val[1]}"
if val1:
values.remove(val1)
self.assertEqual(len(values), 0)
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={
"s1":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
},
"s2":{
SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
}
})
es = pattern_one.expand_sweeps()
self.assertIsInstance(es, list)
self.assertEqual(len(es), 15)
values = [
"s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20",
"s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35",
"s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50",
"s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65",
"s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80",
]
for sweep_vals in es:
self.assertIsInstance(sweep_vals, tuple)
self.assertEqual(len(sweep_vals), 2)
val1 = None
val2 = None
for sweep_val in sweep_vals:
self.assertIsInstance(sweep_val, tuple)
self.assertEqual(len(sweep_val), 2)
if sweep_val[0] == "s1":
val1 = f"s1-{sweep_val[1]}"
if sweep_val[0] == "s2":
val2 = f"s2-{sweep_val[1]}"
if val1 and val2:
values.remove(f"{val1}/{val2}")
self.assertEqual(len(values), 0)
# Test that BaseRecipe instantiation
def testBaseRule(self)->None:
with self.assertRaises(TypeError):
@ -87,7 +160,7 @@ class MeowTests(unittest.TestCase):
pass
def _is_valid_pattern(self, pattern:Any)->None:
pass
FullRule("name", "", "")
FullRule("name", valid_pattern_one, valid_recipe_one)
# Test that create_rule creates a rule from pattern and recipe
def testCreateRule(self)->None:
@ -227,5 +300,3 @@ class MeowTests(unittest.TestCase):
pass
FullTestConductor()
# TODO Test expansion of parameter sweeps

407
tests/test_recipes.py
View File

@ -7,20 +7,25 @@ from multiprocessing import Pipe
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \
PARAMS_FILE, JOB_FILE, RESULT_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, JOB_ERROR, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from core.correctness.validation import valid_job
from core.functionality import get_file_hash, create_job, \
create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml
create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml, \
write_file, lines_to_string
from core.meow import create_rules, create_rule
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, job_func
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from shared import setup, teardown, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
PapermillHandler, papermill_job_func
from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from rules import FileEventJupyterNotebookRule, FileEventPythonRule
from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
COMPLETE_PYTHON_SCRIPT, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK
class JupyterNotebookTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -349,7 +354,7 @@ class JupyterNotebookTests(unittest.TestCase):
}
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -359,7 +364,7 @@ class JupyterNotebookTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -375,25 +380,35 @@ class JupyterNotebookTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
job_func(job_dict)
papermill_job_func(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_dir))
self.assertTrue(os.path.exists(os.path.join(job_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(job_dir, BASE_FILE)))
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(job_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(job_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
# Test jobFunc doesn't execute with no args
def testJobFuncBadArgs(self)->None:
try:
job_func({})
papermill_job_func({})
except Exception:
pass
@ -411,3 +426,363 @@ class PythonTests(unittest.TestCase):
def tearDown(self)->None:
super().tearDown()
teardown()
# Test PythonRecipe can be created
def testPythonRecipeCreationMinimum(self)->None:
PythonRecipe("test_recipe", BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe cannot be created without name
def testPythonRecipeCreationNoName(self)->None:
with self.assertRaises(ValueError):
PythonRecipe("", BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe cannot be created with invalid name
def testPythonRecipeCreationInvalidName(self)->None:
with self.assertRaises(ValueError):
PythonRecipe("@test_recipe", BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe cannot be created with invalid recipe
def testPythonRecipeCreationInvalidRecipe(self)->None:
with self.assertRaises(TypeError):
PythonRecipe("test_recipe", BAREBONES_NOTEBOOK)
# Test PythonRecipe name setup correctly
def testPythonRecipeSetupName(self)->None:
name = "name"
pr = PythonRecipe(name, BAREBONES_PYTHON_SCRIPT)
self.assertEqual(pr.name, name)
# Test PythonRecipe recipe setup correctly
def testPythonRecipeSetupRecipe(self)->None:
pr = PythonRecipe("name", BAREBONES_PYTHON_SCRIPT)
self.assertEqual(pr.recipe, BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe parameters setup correctly
def testPythonRecipeSetupParameters(self)->None:
parameters = {
"a": 1,
"b": True
}
pr = PythonRecipe(
"name", BAREBONES_PYTHON_SCRIPT, parameters=parameters)
self.assertEqual(pr.parameters, parameters)
# Test PythonRecipe requirements setup correctly
def testPythonRecipeSetupRequirements(self)->None:
requirements = {
"a": 1,
"b": True
}
pr = PythonRecipe(
"name", BAREBONES_PYTHON_SCRIPT, requirements=requirements)
self.assertEqual(pr.requirements, requirements)
# Test PythonHandler can be created
def testPythonHandlerMinimum(self)->None:
PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
# Test PythonHandler will handle given events
def testPythonHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
if from_handler_reader.poll(3):
job_dir = from_handler_reader.recv()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
# Test PythonHandler will create enough jobs from single sweep
def testPythonHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={"s":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
}})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
else:
recieving = False
values = [0, 1, 2]
self.assertEqual(len(jobs), 3)
for job_dir in jobs:
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
self.assertIn(JOB_PARAMETERS, job)
self.assertIn("s", job[JOB_PARAMETERS])
if job[JOB_PARAMETERS]["s"] in values:
values.remove(job[JOB_PARAMETERS]["s"])
self.assertEqual(len(values), 0)
# Test PythonHandler will create enough jobs from multiple sweeps
def testPythonHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={
"s1":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
},
"s2":{
SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
}
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
else:
recieving = False
values = [
"s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20",
"s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35",
"s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50",
"s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65",
"s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80",
]
self.assertEqual(len(jobs), 15)
for job_dir in jobs:
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
self.assertIn(JOB_PARAMETERS, job)
val1 = None
val2 = None
if "s1" in job[JOB_PARAMETERS]:
val1 = f"s1-{job[JOB_PARAMETERS]['s1']}"
if "s2" in job[JOB_PARAMETERS]:
val2 = f"s2-{job[JOB_PARAMETERS]['s2']}"
val = None
if val1 and val2:
val = f"{val1}/{val2}"
if val and val in values:
values.remove(val)
self.assertEqual(len(values), 0)
# Test jobFunc performs as expected
def testJobFunc(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
with open(file_path, "w") as f:
f.write("250")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile": result_path
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
rule = create_rule(pattern, recipe)
params_dict = {
"extra":"extra",
"infile":file_path,
"outfile": result_path
}
job_dict = create_job(
JOB_TYPE_PYTHON,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
job_dir = os.path.join(
job_dict[PYTHON_EXECUTION_BASE], job_dict[JOB_ID])
make_dir(job_dir)
meta_file = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_file)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_notebook(APPENDING_NOTEBOOK, base_file)
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
python_job_func(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_dir))
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(result_path))
# Test jobFunc doesn't execute with no args
def testJobFuncBadArgs(self)->None:
try:
python_job_func({})
except Exception:
pass
self.assertEqual(len(os.listdir(TEST_HANDLER_BASE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
# TODO test default parameter function execution

15
tests/test_runner.py
View File

@ -6,7 +6,7 @@ import unittest
from time import sleep
from conductors import LocalPythonConductor
from core.correctness.vars import RESULT_FILE
from core.correctness.vars import get_result_file, JOB_TYPE_PAPERMILL
from core.functionality import make_dir, read_notebook
from core.meow import BaseMonitor, BaseHandler, BaseConductor
from core.runner import MeowRunner
@ -187,7 +187,8 @@ class MeowTests(unittest.TestCase):
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(job_dir)), 5)
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
result = read_notebook(
os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
self.assertIsNotNone(result)
output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
@ -279,7 +280,8 @@ class MeowTests(unittest.TestCase):
mid_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(mid_job_dir)), 5)
result = read_notebook(os.path.join(mid_job_dir, RESULT_FILE))
result = read_notebook(
os.path.join(mid_job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
self.assertIsNotNone(result)
mid_output_path = os.path.join(TEST_MONITOR_BASE, "middle", "A.txt")
@ -293,7 +295,8 @@ class MeowTests(unittest.TestCase):
final_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(final_job_dir)), 5)
result = read_notebook(os.path.join(final_job_dir, RESULT_FILE))
result = read_notebook(os.path.join(final_job_dir,
get_result_file(JOB_TYPE_PAPERMILL)))
self.assertIsNotNone(result)
final_output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
@ -305,8 +308,8 @@ class MeowTests(unittest.TestCase):
self.assertEqual(data,
"Initial Data\nA line from Pattern 1\nA line from Pattern 2")
# TODO sweep tests
# TODO adding tests with numpy
# TODO sweep execution test
# TODO adding tests with numpy or other external dependency
# TODO test getting job cannot handle
# TODO test getting event cannot handle
# TODO test with several matched monitors

13
tests/test_validation.py
View File

@ -9,7 +9,7 @@ from typing import Any, Union
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
setup_debugging, valid_watchdog_event
setup_debugging, valid_watchdog_event, check_callable
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
@ -304,7 +304,7 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(TypeError):
setup_debugging(stream, "1")
#Test watchdog event dict
# Test watchdog event dict
def testWatchdogEventValidation(self)->None:
valid_watchdog_event({
EVENT_TYPE: "test",
@ -336,4 +336,11 @@ class CorrectnessTests(unittest.TestCase):
valid_event({"EVENT_TYPE": "test"})
with self.assertRaises(KeyError):
valid_event({})
valid_event({})
# Test check_callable
def testCheckCallable(self)->None:
check_callable(make_dir)
with self.assertRaises(TypeError):
check_callable("a")