NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

differentiated papermill and python jobs more clearly

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-03 14:47:16 +01:00
parent 72d6b263b7
commit 47f9fe73fa
17 changed files with 853 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
tests/test_conductors.py
View File

@ -4,15 +4,20 @@ import unittest
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \
META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
STATUS_DONE, JOB_TYPE_PAPERMILL, get_base_file, get_result_file, \
get_job_file
from core.functionality import get_file_hash, create_watchdog_event, \
create_job, make_dir, write_yaml, write_notebook
create_job, make_dir, write_yaml, write_notebook, read_yaml, write_file, \
lines_to_string
from core.meow import create_rule
from conductors import LocalPythonConductor
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from recipes.python_recipe import PythonRecipe, python_job_func
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
TEST_JOB_OUTPUT, TEST_HANDLER_BASE
TEST_JOB_OUTPUT, TEST_HANDLER_BASE, COMPLETE_PYTHON_SCRIPT
def failing_func():
@ -32,10 +37,94 @@ class MeowTests(unittest.TestCase):
def testLocalPythonConductorCreation(self)->None:
LocalPythonConductor()
#TODO Test LocalPythonConductor execution criteria
#TODO Test LocalPythonConductor executes valid python jobs
def testLocalPythonConductorValidPythonJob(self)->None:
lpc = LocalPythonConductor()
# Test LocalPythonConductor executes valid jobs
def testLocalPythonConductorValidJob(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
with open(file_path, "w") as f:
f.write("150")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"num":450,
"outfile":result_path
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
rule = create_rule(pattern, recipe)
params_dict = {
"num":450,
"infile":file_path,
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_PYTHON,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
lpc.execute(job_dict)
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor executes valid papermill jobs
def testLocalPythonConductorValidPapermillJob(self)->None:
lpc = LocalPythonConductor()
file_path = os.path.join(TEST_MONITOR_BASE, "test")
@ -67,7 +156,7 @@ class MeowTests(unittest.TestCase):
}
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -77,7 +166,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -89,7 +178,10 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(job_dict)
@ -99,11 +191,22 @@ class MeowTests(unittest.TestCase):
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
@ -140,7 +243,7 @@ class MeowTests(unittest.TestCase):
}
bad_job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -150,7 +253,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
}
)
@ -160,7 +263,7 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
with self.assertRaises(KeyError):
@ -168,7 +271,7 @@ class MeowTests(unittest.TestCase):
# Ensure execution can continue after one failed job
good_job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -178,7 +281,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -190,7 +293,7 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(good_job_dict)
@ -201,10 +304,13 @@ class MeowTests(unittest.TestCase):
output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
@ -235,7 +341,7 @@ class MeowTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,