NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added standardised job creation

Browse Source
This commit is contained in:
PatchOfScotland
2023-04-22 21:48:33 +02:00
parent f306d8b6f2
commit d3eb2dbf9f
15 changed files with 515 additions and 1281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

438
tests/test_conductors.py
View File

@ -4,31 +4,31 @@ import stat
import unittest
from datetime import datetime
from multiprocessing import Pipe
from typing import Dict
from meow_base.core.vars import JOB_TYPE_PYTHON, SHA256, \
JOB_PARAMETERS, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, \
JOB_EVENT, META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \
JOB_EVENT, META_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \
JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \
JOB_CREATE_TIME, JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, \
get_base_file, get_result_file, get_job_file
EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_FILE
from meow_base.conductors import LocalPythonConductor, LocalBashConductor
from meow_base.functionality.file_io import read_file, read_yaml, write_file, \
write_notebook, write_yaml, lines_to_string, make_dir
write_yaml, lines_to_string, make_dir, threadsafe_read_status
from meow_base.functionality.hashing import get_hash
from meow_base.functionality.meow import create_watchdog_event, create_job, \
from meow_base.functionality.meow import create_watchdog_event, create_job_metadata_dict, \
create_rule
from meow_base.functionality.parameterisation import parameterize_bash_script
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from meow_base.recipes.python_recipe import PythonRecipe, python_job_func
from meow_base.recipes.bash_recipe import BashRecipe, assemble_bash_job_script
PapermillHandler
from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler
from meow_base.recipes.bash_recipe import BashRecipe, BashHandler
from shared import TEST_MONITOR_BASE, APPENDING_NOTEBOOK, TEST_JOB_OUTPUT, \
TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, BAREBONES_PYTHON_SCRIPT, \
BAREBONES_NOTEBOOK, COMPLETE_BASH_SCRIPT, BAREBONES_BASH_SCRIPT, \
setup, teardown
setup, teardown, count_non_locks
def failing_func():
raise Exception("bad function")
@ -58,10 +58,16 @@ class PythonTests(unittest.TestCase):
# Test LocalPythonConductor executes valid python jobs
def testLocalPythonConductorValidPythonJob(self)->None:
from_handler_to_runner_reader, from_handler_to_runner_writer = Pipe()
bh = PythonHandler(job_queue_dir=TEST_JOB_QUEUE)
bh.to_runner_job = from_handler_to_runner_writer
conductor_to_test_conductor, conductor_to_test_test = Pipe(duplex=True)
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
lpc.to_runner_job = conductor_to_test_conductor
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
@ -91,37 +97,34 @@ class PythonTests(unittest.TestCase):
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_PYTHON,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
PYTHON_FUNC:python_job_func
}
event = create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
bh.setup_job(event, params_dict)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
lpc.start()
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
# Get valid job
if from_handler_to_runner_reader.poll(3):
job_queue_dir = from_handler_to_runner_reader.recv()
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
# Send it to conductor
if conductor_to_test_test.poll(3):
_ = conductor_to_test_test.recv()
conductor_to_test_test.send(job_queue_dir)
lpc.execute(job_dir)
# Wait for job to complete
if conductor_to_test_test.poll(3):
_ = conductor_to_test_test.recv()
conductor_to_test_test.send(1)
self.assertFalse(os.path.exists(job_dir))
job_output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
job_output_dir = job_queue_dir.replace(TEST_JOB_QUEUE, TEST_JOB_OUTPUT)
self.assertFalse(os.path.exists(job_queue_dir))
self.assertTrue(os.path.exists(job_output_dir))
meta_path = os.path.join(job_output_dir, META_FILE)
@ -129,26 +132,33 @@ class PythonTests(unittest.TestCase):
status = read_yaml(meta_path)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
print(status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_job_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_result_file(JOB_TYPE_PYTHON))))
print(os.listdir(job_output_dir))
self.assertEqual(count_non_locks(job_output_dir), 4)
for f in [META_FILE, "recipe.py", "output.log", "job.sh"]:
self.assertTrue(os.path.exists(os.path.join(job_output_dir, f)))
self.assertTrue(os.path.exists(result_path))
result = read_file(result_path)
self.assertEqual(result, "25293.75")
# Test LocalPythonConductor executes valid papermill jobs
def testLocalPythonConductorValidPapermillJob(self)->None:
from_handler_to_runner_reader, from_handler_to_runner_writer = Pipe()
bh = PapermillHandler(job_queue_dir=TEST_JOB_QUEUE)
bh.to_runner_job = from_handler_to_runner_writer
conductor_to_test_conductor, conductor_to_test_test = Pipe(duplex=True)
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
lpc.to_runner_job = conductor_to_test_conductor
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
@ -178,38 +188,34 @@ class PythonTests(unittest.TestCase):
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
PYTHON_FUNC:papermill_job_func
}
event = create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
bh.setup_job(event, params_dict)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
lpc.start()
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
# Get valid job
if from_handler_to_runner_reader.poll(3):
job_queue_dir = from_handler_to_runner_reader.recv()
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
# Send it to conductor
if conductor_to_test_test.poll(3):
_ = conductor_to_test_test.recv()
conductor_to_test_test.send(job_queue_dir)
lpc.execute(job_dir)
# Wait for job to complete
if conductor_to_test_test.poll(3):
_ = conductor_to_test_test.recv()
conductor_to_test_test.send(1)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
job_output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
job_output_dir = job_queue_dir.replace(TEST_JOB_QUEUE, TEST_JOB_OUTPUT)
self.assertFalse(os.path.exists(job_queue_dir))
self.assertTrue(os.path.exists(job_output_dir))
meta_path = os.path.join(job_output_dir, META_FILE)
@ -219,139 +225,13 @@ class PythonTests(unittest.TestCase):
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor does not execute jobs with bad arguments
def testLocalPythonConductorBadArgs(self)->None:
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
with open(file_path, "w") as f:
f.write("Data")
file_hash = get_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":result_path
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
params_dict = {
"extra":"extra",
"infile":file_path,
"outfile":result_path
}
bad_job_dict = create_job(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict
}
)
bad_job_dir = os.path.join(TEST_JOB_QUEUE, bad_job_dict[JOB_ID])
make_dir(bad_job_dir)
bad_param_file = os.path.join(bad_job_dir, PARAMS_FILE)
write_yaml(params_dict, bad_param_file)
bad_meta_path = os.path.join(bad_job_dir, META_FILE)
write_yaml(bad_job_dict, bad_meta_path)
bad_base_file = os.path.join(bad_job_dir,
get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, bad_base_file)
lpc.execute(bad_job_dir)
bad_output_dir = os.path.join(TEST_JOB_OUTPUT, bad_job_dict[JOB_ID])
self.assertFalse(os.path.exists(bad_job_dir))
self.assertTrue(os.path.exists(bad_output_dir))
bad_meta_path = os.path.join(bad_output_dir, META_FILE)
self.assertTrue(os.path.exists(bad_meta_path))
bad_job = read_yaml(bad_meta_path)
self.assertIsInstance(bad_job, dict)
self.assertIn(JOB_ERROR, bad_job)
# Ensure execution can continue after one failed job
good_job_dict = create_job(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
PYTHON_FUNC:papermill_job_func
}
)
good_job_dir = os.path.join(TEST_JOB_QUEUE, good_job_dict[JOB_ID])
make_dir(good_job_dir)
good_param_file = os.path.join(good_job_dir, PARAMS_FILE)
write_yaml(params_dict, good_param_file)
good_meta_path = os.path.join(good_job_dir, META_FILE)
write_yaml(good_job_dict, good_meta_path)
good_base_file = os.path.join(good_job_dir,
get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, good_base_file)
lpc.execute(good_job_dir)
good_job_dir = os.path.join(TEST_JOB_QUEUE, good_job_dict[JOB_ID])
self.assertFalse(os.path.exists(good_job_dir))
good_job_output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
self.assertTrue(os.path.exists(good_job_output_dir))
self.assertTrue(os.path.exists(
os.path.join(good_job_output_dir, META_FILE)))
self.assertTrue(os.path.exists(
os.path.join(good_job_output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(good_job_output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(good_job_output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(good_job_output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertEqual(count_non_locks(job_output_dir), 4)
for f in [META_FILE, JOB_FILE, "result.ipynb", "recipe.ipynb"]:
self.assertTrue(os.path.exists(os.path.join(job_output_dir, f)))
self.assertTrue(os.path.exists(result_path))
result = read_file(result_path)
self.assertEqual(result, "Data\nextra")
# Test LocalPythonConductor does not execute jobs with missing metafile
def testLocalPythonConductorMissingMetafile(self)->None:
@ -382,7 +262,7 @@ class PythonTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
job_dict = create_job(
job_dict = create_job_metadata_dict(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
@ -418,77 +298,6 @@ class PythonTests(unittest.TestCase):
"Recieved incorrectly setup job.\n\n[Errno 2] No such file or "
f"directory: 'test_job_queue_dir{os.path.sep}{job_dict[JOB_ID]}{os.path.sep}job.yml'")
# Test LocalPythonConductor does not execute jobs with bad functions
def testLocalPythonConductorBadFunc(self)->None:
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output", "test")
with open(file_path, "w") as f:
f.write("Data")
file_hash = get_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":result_path
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
rule = create_rule(pattern, recipe)
params = {
"extra":"extra",
"infile":file_path,
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params,
PYTHON_FUNC:failing_func,
}
)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params, param_file)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
lpc.execute(job_dir)
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertFalse(os.path.exists(job_dir))
self.assertTrue(os.path.exists(output_dir))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
job = read_yaml(meta_path)
self.assertIsInstance(job, dict)
self.assertIn(JOB_ERROR, job)
# Test LocalPythonConductor does not execute jobs with invalid metafile
def testLocalPythonConductorInvalidMetafile(self)->None:
lpc = LocalPythonConductor(
@ -518,7 +327,7 @@ class PythonTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
job_dict = create_job(
job_dict = create_job_metadata_dict(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
@ -586,7 +395,7 @@ class PythonTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
job_dict = create_job(
job_dict = create_job_metadata_dict(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
@ -746,10 +555,16 @@ class BashTests(unittest.TestCase):
# Test LocalBashConductor executes valid bash jobs
def testLocalBashConductorValidBashJob(self)->None:
from_handler_to_runner_reader, from_handler_to_runner_writer = Pipe()
bh = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
bh.to_runner_job = from_handler_to_runner_writer
conductor_to_test_conductor, conductor_to_test_test = Pipe(duplex=True)
lpc = LocalBashConductor(
job_queue_dir=TEST_JOB_QUEUE,
job_output_dir=TEST_JOB_OUTPUT
)
lpc.to_runner_job = conductor_to_test_conductor
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
@ -779,44 +594,34 @@ class BashTests(unittest.TestCase):
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_BASH,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
}
event = create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
bh.setup_job(event, params_dict)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
lpc.start()
base_script = parameterize_bash_script(
COMPLETE_BASH_SCRIPT, params_dict
)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))
write_file(lines_to_string(base_script), base_file)
st = os.stat(base_file)
os.chmod(base_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
# Get valid job
if from_handler_to_runner_reader.poll(3):
job_queue_dir = from_handler_to_runner_reader.recv()
job_script = assemble_bash_job_script()
job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))
write_file(lines_to_string(job_script), job_file)
st = os.stat(job_file)
os.chmod(job_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
# Send it to conductor
if conductor_to_test_test.poll(3):
_ = conductor_to_test_test.recv()
conductor_to_test_test.send(job_queue_dir)
lpc.execute(job_dir)
# Wait for job to complete
if conductor_to_test_test.poll(3):
_ = conductor_to_test_test.recv()
conductor_to_test_test.send(1)
self.assertFalse(os.path.exists(job_dir))
job_output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
job_output_dir = job_queue_dir.replace(TEST_JOB_QUEUE, TEST_JOB_OUTPUT)
self.assertFalse(os.path.exists(job_queue_dir))
self.assertTrue(os.path.exists(job_output_dir))
meta_path = os.path.join(job_output_dir, META_FILE)
@ -824,15 +629,24 @@ class BashTests(unittest.TestCase):
status = read_yaml(meta_path)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertNotIn(JOB_ERROR, status)
self.assertEqual(count_non_locks(job_output_dir), 3)
for f in [META_FILE, JOB_FILE]:
self.assertTrue(os.path.exists(os.path.join(job_output_dir, f)))
job = threadsafe_read_status(os.path.join(job_output_dir, META_FILE))
self.assertTrue(os.path.exists(os.path.join(job_output_dir, job["tmp script command"])))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_base_file(JOB_TYPE_BASH))))
os.path.join(job_output_dir, )))
self.assertTrue(os.path.exists(
os.path.join(job_output_dir, get_job_file(JOB_TYPE_BASH))))
os.path.join(job_output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(result_path))
result = read_file(result_path)
self.assertEqual(result, "25293\n")
# Test LocalBashConductor does not execute jobs with missing metafile
def testLocalBashConductorMissingMetafile(self)->None:
@ -869,7 +683,7 @@ class BashTests(unittest.TestCase):
"outfile":result_path
}
job_dict = create_job(
job_dict = create_job_metadata_dict(
JOB_TYPE_BASH,
create_watchdog_event(
file_path,
@ -935,7 +749,7 @@ class BashTests(unittest.TestCase):
"outfile":result_path
}
job_dict = create_job(
job_dict = create_job_metadata_dict(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
@ -958,7 +772,7 @@ class BashTests(unittest.TestCase):
base_script = parameterize_bash_script(
COMPLETE_BASH_SCRIPT, params_dict
)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))
base_file = os.path.join(job_dir, JOB_FILE)
write_file(lines_to_string(base_script), base_file)
st = os.stat(base_file)
os.chmod(base_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
@ -967,7 +781,7 @@ class BashTests(unittest.TestCase):
"#!/bin/bash",
"echo Does Nothing"
]
job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))
job_file = os.path.join(job_dir, JOB_FILE)
write_file(lines_to_string(job_script), job_file)
st = os.stat(job_file)
os.chmod(job_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
@ -1019,7 +833,7 @@ class BashTests(unittest.TestCase):
"outfile":result_path
}
job_dict = create_job(
job_dict = create_job_metadata_dict(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
@ -1088,7 +902,7 @@ class BashTests(unittest.TestCase):
"outfile":result_path
}
job_dict = create_job(
job_dict = create_job_metadata_dict(
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,