NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformted jobs being passed to conductors so they only get a job directory and have to read the definitions from the appropriate files

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-09 15:22:26 +01:00
parent 1eb022f79e
commit a2df62c693
19 changed files with 528 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tests/shared.py
View File

@ -5,23 +5,31 @@ Author(s): David Marchant
"""
import os
from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from core.functionality import make_dir, rmtree
# testing
TEST_DIR = "test_files"
TEST_MONITOR_BASE = "test_monitor_base"
TEST_HANDLER_BASE = "test_handler_base"
TEST_JOB_QUEUE = "test_job_queue_dir"
TEST_JOB_OUTPUT = "test_job_output"
def setup():
make_dir(TEST_DIR, ensure_clean=True)
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
make_dir(TEST_HANDLER_BASE, ensure_clean=True)
make_dir(TEST_JOB_QUEUE, ensure_clean=True)
make_dir(TEST_JOB_OUTPUT, ensure_clean=True)
make_dir(DEFAULT_JOB_OUTPUT_DIR, ensure_clean=True)
make_dir(DEFAULT_JOB_QUEUE_DIR, ensure_clean=True)
def teardown():
rmtree(TEST_DIR)
rmtree(TEST_MONITOR_BASE)
rmtree(TEST_HANDLER_BASE)
rmtree(TEST_JOB_QUEUE)
rmtree(TEST_JOB_OUTPUT)
rmtree(DEFAULT_JOB_OUTPUT_DIR)
rmtree(DEFAULT_JOB_QUEUE_DIR)
rmtree("first")
# Recipe funcs

230
tests/test_conductors.py
View File

@ -5,7 +5,7 @@ import unittest
from typing import Dict
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \
JOB_HASH, PYTHON_FUNC, JOB_ID, \
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
STATUS_DONE, JOB_TYPE_PAPERMILL, get_base_file, get_result_file, \
get_job_file
@ -19,7 +19,7 @@ from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from recipes.python_recipe import PythonRecipe, python_job_func
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
TEST_JOB_OUTPUT, TEST_HANDLER_BASE, COMPLETE_PYTHON_SCRIPT
TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT
def failing_func():
@ -41,7 +41,10 @@ class MeowTests(unittest.TestCase):
# Test LocalPythonConductor executes valid python jobs
def testLocalPythonConductorValidPythonJob(self)->None:
lpc = LocalPythonConductor()
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
@ -82,13 +85,11 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
PYTHON_FUNC:python_job_func
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
@ -100,14 +101,14 @@ class MeowTests(unittest.TestCase):
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
lpc.execute(job_dict)
lpc.execute(job_dir)
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
job_output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_output_dir))
meta_path = os.path.join(output_dir, META_FILE)
meta_path = os.path.join(job_output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, Dict)
@ -116,18 +117,22 @@ class MeowTests(unittest.TestCase):
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
os.path.join(job_output_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PYTHON))))
os.path.join(job_output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PYTHON))))
os.path.join(job_output_dir, get_job_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_result_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor executes valid papermill jobs
def testLocalPythonConductorValidPapermillJob(self)->None:
lpc = LocalPythonConductor()
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
@ -168,13 +173,11 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
PYTHON_FUNC:papermill_job_func
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
@ -186,16 +189,15 @@ class MeowTests(unittest.TestCase):
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(job_dict)
lpc.execute(job_dir)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
job_output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_output_dir))
meta_path = os.path.join(output_dir, META_FILE)
meta_path = os.path.join(job_output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, Dict)
@ -203,18 +205,22 @@ class MeowTests(unittest.TestCase):
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
os.path.join(job_output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
os.path.join(job_output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
os.path.join(job_output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor does not execute jobs with bad arguments
def testLocalPythonConductorBadArgs(self)->None:
lpc = LocalPythonConductor()
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
@ -255,21 +261,34 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:papermill_job_func,
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, bad_job_dict[JOB_ID])
make_dir(job_dir)
bad_job_dir = os.path.join(TEST_JOB_QUEUE, bad_job_dict[JOB_ID])
make_dir(bad_job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
bad_param_file = os.path.join(bad_job_dir, PARAMS_FILE)
write_yaml(params_dict, bad_param_file)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
bad_meta_path = os.path.join(bad_job_dir, META_FILE)
write_yaml(bad_job_dict, bad_meta_path)
with self.assertRaises(KeyError):
lpc.execute(bad_job_dict)
bad_base_file = os.path.join(bad_job_dir,
get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, bad_base_file)
lpc.execute(bad_job_dir)
bad_output_dir = os.path.join(TEST_JOB_OUTPUT, bad_job_dict[JOB_ID])
self.assertFalse(os.path.exists(bad_job_dir))
self.assertTrue(os.path.exists(bad_output_dir))
bad_meta_path = os.path.join(bad_output_dir, META_FILE)
self.assertTrue(os.path.exists(bad_meta_path))
bad_job = read_yaml(bad_meta_path)
self.assertIsInstance(bad_job, dict)
self.assertIn(JOB_ERROR, bad_job)
# Ensure execution can continue after one failed job
good_job_dict = create_job(
@ -283,42 +302,50 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
PYTHON_FUNC:papermill_job_func
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, good_job_dict[JOB_ID])
make_dir(job_dir)
good_job_dir = os.path.join(TEST_JOB_QUEUE, good_job_dict[JOB_ID])
make_dir(good_job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
good_param_file = os.path.join(good_job_dir, PARAMS_FILE)
write_yaml(params_dict, good_param_file)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
good_meta_path = os.path.join(good_job_dir, META_FILE)
write_yaml(good_job_dict, good_meta_path)
lpc.execute(good_job_dict)
good_base_file = os.path.join(good_job_dir,
get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, good_base_file)
job_dir = os.path.join(TEST_HANDLER_BASE, good_job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
lpc.execute(good_job_dir)
good_job_dir = os.path.join(TEST_JOB_QUEUE, good_job_dict[JOB_ID])
self.assertFalse(os.path.exists(good_job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
good_job_output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
self.assertTrue(os.path.exists(good_job_output_dir))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
os.path.join(good_job_output_dir, META_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
os.path.join(good_job_output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
os.path.join(good_job_output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(good_job_output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(good_job_output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor does not execute jobs with bad functions
def testLocalPythonConductorBadFunc(self)->None:
lpc = LocalPythonConductor()
# Test LocalPythonConductor does not execute jobs with missing metafile
def testLocalPythonConductorMissingMetafile(self)->None:
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
@ -361,8 +388,83 @@ class MeowTests(unittest.TestCase):
}
)
with self.assertRaises(Exception):
lpc.execute(job_dict)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
with self.assertRaises(FileNotFoundError):
lpc.execute(job_dir)
# Test LocalPythonConductor does not execute jobs with bad functions
def testLocalPythonConductorBadFunc(self)->None:
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
with open(file_path, "w") as f:
f.write("Data")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":result_path
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
params = {
"extra":"extra",
"infile":file_path,
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params,
JOB_HASH: file_hash,
PYTHON_FUNC:failing_func,
}
)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params, param_file)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
lpc.execute(job_dir)
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
self.assertTrue(os.path.exists(output_dir))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
job = read_yaml(meta_path)
self.assertIsInstance(job, dict)
self.assertIn(JOB_ERROR, job)
# TODO test job status funcs
# TODO test mangled status file reads

6
tests/test_functionality.py
View File

@ -10,7 +10,7 @@ from time import sleep
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, JOB_EVENT, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
@ -335,9 +335,7 @@ class CorrectnessTests(unittest.TestCase):
"outfile":"result_path"
},
JOB_HASH: "file_hash",
PYTHON_FUNC:max,
PYTHON_OUTPUT_DIR:"output",
PYTHON_EXECUTION_BASE:"execution"
PYTHON_FUNC:max
}
)

2
tests/test_meow.py
View File

@ -292,7 +292,7 @@ class MeowTests(unittest.TestCase):
TestConductor()
class FullTestConductor(BaseConductor):
def execute(self, job:Dict[str,Any])->None:
def execute(self, job_dir:str)->None:
pass
def valid_execute_criteria(self, job:Dict[str,Any]

69
tests/test_recipes.py
View File

@ -9,7 +9,7 @@ from typing import Dict
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, JOB_ERROR, \
META_FILE, JOB_ERROR, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from core.correctness.validation import valid_job
@ -23,7 +23,7 @@ from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from rules import FileEventJupyterNotebookRule, FileEventPythonRule
from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
COMPLETE_PYTHON_SCRIPT, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
COMPLETE_PYTHON_SCRIPT, TEST_JOB_QUEUE, TEST_MONITOR_BASE, \
TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK
@ -112,18 +112,12 @@ class JupyterNotebookTests(unittest.TestCase):
# Test PapermillHandler can be created
def testPapermillHanderMinimum(self)->None:
PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
# Test PapermillHandler will handle given events
def testPapermillHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
@ -172,10 +166,7 @@ class JupyterNotebookTests(unittest.TestCase):
# Test PapermillHandler will create enough jobs from single sweep
def testPapermillHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
@ -240,10 +231,7 @@ class JupyterNotebookTests(unittest.TestCase):
# Test PapermillHandler will create enough jobs from multiple sweeps
def testPapermillHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
@ -365,14 +353,11 @@ class JupyterNotebookTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
PYTHON_FUNC:papermill_job_func
}
)
job_dir = os.path.join(
job_dict[PYTHON_EXECUTION_BASE], job_dict[JOB_ID])
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
meta_file = os.path.join(job_dir, META_FILE)
@ -384,9 +369,9 @@ class JupyterNotebookTests(unittest.TestCase):
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
papermill_job_func(job_dict)
papermill_job_func(job_dir)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_dir))
meta_path = os.path.join(job_dir, META_FILE)
@ -413,7 +398,7 @@ class JupyterNotebookTests(unittest.TestCase):
except Exception:
pass
self.assertEqual(len(os.listdir(TEST_HANDLER_BASE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_QUEUE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
#TODO Test handling criteria function
@ -479,18 +464,12 @@ class PythonTests(unittest.TestCase):
# Test PythonHandler can be created
def testPythonHandlerMinimum(self)->None:
PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
# Test PythonHandler will handle given events
def testPythonHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
@ -539,10 +518,7 @@ class PythonTests(unittest.TestCase):
# Test PythonHandler will create enough jobs from single sweep
def testPythonHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
@ -607,10 +583,7 @@ class PythonTests(unittest.TestCase):
# Test PythonHandler will create enough jobs from multiple sweeps
def testPythonHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
@ -732,14 +705,11 @@ class PythonTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
PYTHON_FUNC:python_job_func
}
)
job_dir = os.path.join(
job_dict[PYTHON_EXECUTION_BASE], job_dict[JOB_ID])
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
meta_file = os.path.join(job_dir, META_FILE)
@ -752,9 +722,8 @@ class PythonTests(unittest.TestCase):
write_notebook(APPENDING_NOTEBOOK, base_file)
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
python_job_func(job_dict)
python_job_func(job_dir)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_dir))
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
@ -782,7 +751,7 @@ class PythonTests(unittest.TestCase):
except Exception:
pass
self.assertEqual(len(os.listdir(TEST_HANDLER_BASE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_QUEUE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
# TODO test default parameter function execution

81
tests/test_runner.py
View File

@ -16,8 +16,8 @@ from recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe
from recipes.python_recipe import PythonHandler, PythonRecipe
from shared import setup, teardown, \
TEST_HANDLER_BASE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, \
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, \
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR
class MeowTests(unittest.TestCase):
@ -28,16 +28,15 @@ class MeowTests(unittest.TestCase):
def tearDown(self)->None:
super().tearDown()
teardown()
# Test MeowRunner creation
def testMeowRunnerSetup(self)->None:
monitor_one = WatchdogMonitor(TEST_MONITOR_BASE, {}, {})
monitor_two = WatchdogMonitor(TEST_MONITOR_BASE, {}, {})
monitors = [ monitor_one, monitor_two ]
handler_one = PapermillHandler(TEST_HANDLER_BASE, TEST_JOB_OUTPUT)
handler_two = PapermillHandler(TEST_HANDLER_BASE, TEST_JOB_OUTPUT)
handler_one = PapermillHandler()
handler_two = PapermillHandler()
handlers = [ handler_one, handler_two ]
conductor_one = LocalPythonConductor()
@ -117,6 +116,48 @@ class MeowTests(unittest.TestCase):
self.assertIsInstance(runner.conductors, list)
for conductor in runner.conductors:
self.assertIsInstance(conductor, BaseConductor)
# Test meow runner directory overrides
def testMeowRunnerDirOverridesSetup(self)->None:
monitor_one = WatchdogMonitor(TEST_MONITOR_BASE, {}, {})
original_queue_dir = os.path.join(TEST_DIR, "original_queue")
original_output_dir = os.path.join(TEST_DIR, "original_output")
overridden_queue_dir = os.path.join(TEST_DIR, "overridden_queue")
overridden_output_dir = os.path.join(TEST_DIR, "overridden_output")
handler_one = PapermillHandler(job_queue_dir=original_queue_dir)
conductor_one = LocalPythonConductor(
job_queue_dir=original_queue_dir,
job_output_dir=original_output_dir
)
self.assertTrue(os.path.exists(original_queue_dir))
self.assertTrue(os.path.exists(original_output_dir))
self.assertFalse(os.path.exists(overridden_queue_dir))
self.assertFalse(os.path.exists(overridden_output_dir))
self.assertEqual(handler_one.job_queue_dir, original_queue_dir)
self.assertEqual(conductor_one.job_queue_dir, original_queue_dir)
self.assertEqual(conductor_one.job_output_dir, original_output_dir)
MeowRunner(
monitor_one,
handler_one,
conductor_one,
job_queue_dir=overridden_queue_dir,
job_output_dir=overridden_output_dir
)
self.assertTrue(os.path.exists(original_queue_dir))
self.assertTrue(os.path.exists(original_output_dir))
self.assertTrue(os.path.exists(overridden_queue_dir))
self.assertTrue(os.path.exists(overridden_output_dir))
self.assertEqual(handler_one.job_queue_dir, overridden_queue_dir)
self.assertEqual(conductor_one.job_queue_dir, overridden_queue_dir)
self.assertEqual(conductor_one.job_output_dir, overridden_output_dir)
# Test single meow papermill job execution
def testMeowRunnerPapermillExecution(self)->None:
@ -148,11 +189,10 @@ class MeowTests(unittest.TestCase):
recipes,
settletime=1
),
PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
),
PapermillHandler(),
LocalPythonConductor(),
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT,
print=runner_debug_stream,
logging=3
)
@ -243,10 +283,11 @@ class MeowTests(unittest.TestCase):
settletime=1
),
PapermillHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
job_queue_dir=TEST_JOB_QUEUE,
),
LocalPythonConductor(),
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT,
print=runner_debug_stream,
logging=3
)
@ -346,10 +387,11 @@ class MeowTests(unittest.TestCase):
settletime=1
),
PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
job_queue_dir=TEST_JOB_QUEUE
),
LocalPythonConductor(),
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT,
print=runner_debug_stream,
logging=3
)
@ -405,7 +447,7 @@ class MeowTests(unittest.TestCase):
self.assertTrue(os.path.exists(output_path))
output = read_file(os.path.join(output_path))
self.assertEqual(output, "12505000.0")
# Test meow python job chaining within runner
def testMeowRunnerLinkedPythonExecution(self)->None:
pattern_one = FileEventPattern(
@ -448,10 +490,11 @@ class MeowTests(unittest.TestCase):
settletime=1
),
PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
job_queue_dir=TEST_JOB_QUEUE
),
LocalPythonConductor(),
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT,
print=runner_debug_stream,
logging=3
)
@ -540,7 +583,7 @@ class MeowTests(unittest.TestCase):
self.assertTrue(os.path.exists(final_output_path))
final_output = read_file(os.path.join(final_output_path))
self.assertEqual(final_output, "2146.5625")
# TODO sweep execution test
# TODO adding tests with numpy or other external dependency
# TODO test getting job cannot handle
@ -551,3 +594,5 @@ class MeowTests(unittest.TestCase):
# TODO test with several mismatched handlers
# TODO test with several matched conductors
# TODO test with several mismatched conductors
# TODO tests runner job queue dir
# TODO tests runner job output dir

13
tests/test_validation.py
View File

@ -8,7 +8,7 @@ from typing import Any, Union
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
valid_dir_path, valid_non_existing_path, valid_event, valid_job, \
setup_debugging, valid_watchdog_event, check_callable
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
@ -215,21 +215,22 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(ValueError):
valid_existing_file_path(dir_path, SHA256)
# Test valid_existing_dir_path can find directories, or not
def testValidExistingDirPath(self)->None:
valid_existing_dir_path(TEST_MONITOR_BASE)
# Test valid_dir_path can find directories, or not
def testValidDirPath(self)->None:
valid_dir_path(TEST_MONITOR_BASE)
valid_dir_path(TEST_MONITOR_BASE, must_exist=False)
dir_path = os.path.join(TEST_MONITOR_BASE, "dir")
with self.assertRaises(FileNotFoundError):
valid_existing_dir_path("not_existing_"+dir_path, SHA256)
valid_dir_path("not_existing_"+dir_path, must_exist=True)
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
with open(file_path, 'w') as hashed_file:
hashed_file.write("Some data\n")
with self.assertRaises(ValueError):
valid_existing_dir_path(file_path, SHA256)
valid_dir_path(file_path)
# Test valid_non_existing_path can find existing paths, or not
def testValidNonExistingPath(self)->None: