split up functionality tests into different classes per file
This commit is contained in:
PatchOfScotland
@ -32,7 +32,7 @@ from recipes import JupyterNotebookRecipe
|
|||||||
from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \
|
from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \
|
||||||
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
|
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class DebugTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self)->None:
|
||||||
super().setUp()
|
super().setUp()
|
||||||
setup()
|
setup()
|
||||||
@ -41,385 +41,15 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
super().tearDown()
|
super().tearDown()
|
||||||
teardown()
|
teardown()
|
||||||
|
|
||||||
# Test that generate_id creates unique ids
|
|
||||||
def testGenerateIDWorking(self)->None:
|
|
||||||
id = generate_id()
|
|
||||||
self.assertEqual(len(id), 16)
|
|
||||||
for i in range(len(id)):
|
|
||||||
self.assertIn(id[i], CHAR_UPPERCASE+CHAR_LOWERCASE)
|
|
||||||
|
|
||||||
# In extrememly rare cases this may fail due to randomness in algorithm
|
class FileIoTests(unittest.TestCase):
|
||||||
new_id = generate_id(existing_ids=[id])
|
def setUp(self)->None:
|
||||||
self.assertNotEqual(id, new_id)
|
super().setUp()
|
||||||
|
setup()
|
||||||
|
|
||||||
another_id = generate_id(length=32)
|
def tearDown(self)->None:
|
||||||
self.assertEqual(len(another_id), 32)
|
super().tearDown()
|
||||||
|
teardown()
|
||||||
again_id = generate_id(charset="a")
|
|
||||||
for i in range(len(again_id)):
|
|
||||||
self.assertIn(again_id[i], "a")
|
|
||||||
|
|
||||||
with self.assertRaises(ValueError):
|
|
||||||
generate_id(length=2, charset="a", existing_ids=["aa"])
|
|
||||||
|
|
||||||
prefix_id = generate_id(length=4, prefix="Test")
|
|
||||||
self.assertEqual(prefix_id, "Test")
|
|
||||||
|
|
||||||
prefix_id = generate_id(prefix="Test")
|
|
||||||
self.assertEqual(len(prefix_id), 16)
|
|
||||||
self.assertTrue(prefix_id.startswith("Test"))
|
|
||||||
|
|
||||||
# Test that wait can wait on multiple pipes
|
|
||||||
def testWaitPipes(self)->None:
|
|
||||||
pipe_one_reader, pipe_one_writer = Pipe()
|
|
||||||
pipe_two_reader, pipe_two_writer = Pipe()
|
|
||||||
|
|
||||||
inputs = [
|
|
||||||
pipe_one_reader, pipe_two_reader
|
|
||||||
]
|
|
||||||
|
|
||||||
pipe_one_writer.send(1)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(pipe_one_reader, readables)
|
|
||||||
self.assertEqual(len(readables), 1)
|
|
||||||
msg = readables[0].recv()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
|
|
||||||
pipe_one_writer.send(1)
|
|
||||||
pipe_two_writer.send(2)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(pipe_one_reader, readables)
|
|
||||||
self.assertIn(pipe_two_reader, readables)
|
|
||||||
self.assertEqual(len(readables), 2)
|
|
||||||
for readable in readables:
|
|
||||||
if readable == pipe_one_reader:
|
|
||||||
msg = readable.recv()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
elif readable == pipe_two_reader:
|
|
||||||
msg = readable.recv()
|
|
||||||
self.assertEqual(msg, 2)
|
|
||||||
|
|
||||||
# Test that wait can wait on multiple queues
|
|
||||||
def testWaitQueues(self)->None:
|
|
||||||
queue_one = Queue()
|
|
||||||
queue_two = Queue()
|
|
||||||
|
|
||||||
inputs = [
|
|
||||||
queue_one, queue_two
|
|
||||||
]
|
|
||||||
|
|
||||||
queue_one.put(1)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(queue_one, readables)
|
|
||||||
self.assertEqual(len(readables), 1)
|
|
||||||
msg = readables[0].get()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
|
|
||||||
queue_one.put(1)
|
|
||||||
queue_two.put(2)
|
|
||||||
sleep(0.1)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(queue_one, readables)
|
|
||||||
self.assertIn(queue_two, readables)
|
|
||||||
self.assertEqual(len(readables), 2)
|
|
||||||
for readable in readables:
|
|
||||||
if readable == queue_one:
|
|
||||||
msg = readable.get()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
elif readable == queue_two:
|
|
||||||
msg = readable.get()
|
|
||||||
self.assertEqual(msg, 2)
|
|
||||||
|
|
||||||
# Test that wait can wait on multiple pipes and queues
|
|
||||||
def testWaitPipesAndQueues(self)->None:
|
|
||||||
pipe_one_reader, pipe_one_writer = Pipe()
|
|
||||||
pipe_two_reader, pipe_two_writer = Pipe()
|
|
||||||
queue_one = Queue()
|
|
||||||
queue_two = Queue()
|
|
||||||
|
|
||||||
inputs = [
|
|
||||||
pipe_one_reader, pipe_two_reader, queue_one, queue_two
|
|
||||||
]
|
|
||||||
|
|
||||||
pipe_one_writer.send(1)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(pipe_one_reader, readables)
|
|
||||||
self.assertEqual(len(readables), 1)
|
|
||||||
msg = readables[0].recv()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
|
|
||||||
pipe_one_writer.send(1)
|
|
||||||
pipe_two_writer.send(2)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(pipe_one_reader, readables)
|
|
||||||
self.assertIn(pipe_two_reader, readables)
|
|
||||||
self.assertEqual(len(readables), 2)
|
|
||||||
for readable in readables:
|
|
||||||
if readable == pipe_one_reader:
|
|
||||||
msg = readable.recv()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
if readable == pipe_two_reader:
|
|
||||||
msg = readable.recv()
|
|
||||||
self.assertEqual(msg, 2)
|
|
||||||
|
|
||||||
queue_one.put(1)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(queue_one, readables)
|
|
||||||
self.assertEqual(len(readables), 1)
|
|
||||||
msg = readables[0].get()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
|
|
||||||
queue_one.put(1)
|
|
||||||
queue_two.put(2)
|
|
||||||
sleep(0.1)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(queue_one, readables)
|
|
||||||
self.assertIn(queue_two, readables)
|
|
||||||
self.assertEqual(len(readables), 2)
|
|
||||||
for readable in readables:
|
|
||||||
if readable == queue_one:
|
|
||||||
msg = readable.get()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
elif readable == queue_two:
|
|
||||||
msg = readable.get()
|
|
||||||
self.assertEqual(msg, 2)
|
|
||||||
|
|
||||||
queue_one.put(1)
|
|
||||||
pipe_one_writer.send(1)
|
|
||||||
sleep(0.1)
|
|
||||||
readables = wait(inputs)
|
|
||||||
|
|
||||||
self.assertIn(queue_one, readables)
|
|
||||||
self.assertIn(pipe_one_reader, readables)
|
|
||||||
self.assertEqual(len(readables), 2)
|
|
||||||
for readable in readables:
|
|
||||||
if readable == queue_one:
|
|
||||||
msg = readable.get()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
elif readable == pipe_one_reader:
|
|
||||||
msg = readable.recv()
|
|
||||||
self.assertEqual(msg, 1)
|
|
||||||
|
|
||||||
# Test that get_file_hash produces the expected hash
|
|
||||||
def testGetFileHashSha256(self)->None:
|
|
||||||
file_path = os.path.join(TEST_MONITOR_BASE, "hased_file.txt")
|
|
||||||
with open(file_path, 'w') as hashed_file:
|
|
||||||
hashed_file.write("Some data\n")
|
|
||||||
expected_hash = \
|
|
||||||
"8557122088c994ba8aa5540ccbb9a3d2d8ae2887046c2db23d65f40ae63abade"
|
|
||||||
|
|
||||||
hash = get_file_hash(file_path, SHA256)
|
|
||||||
self.assertEqual(hash, expected_hash)
|
|
||||||
|
|
||||||
# Test that get_file_hash raises on a missing file
|
|
||||||
def testGetFileHashSha256NoFile(self)->None:
|
|
||||||
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
|
|
||||||
|
|
||||||
with self.assertRaises(FileNotFoundError):
|
|
||||||
get_file_hash(file_path, SHA256)
|
|
||||||
|
|
||||||
# Test that parameterize_jupyter_notebook parameterises given notebook
|
|
||||||
def testParameteriseNotebook(self)->None:
|
|
||||||
pn = parameterize_jupyter_notebook(
|
|
||||||
COMPLETE_NOTEBOOK, {})
|
|
||||||
|
|
||||||
self.assertEqual(pn, COMPLETE_NOTEBOOK)
|
|
||||||
|
|
||||||
pn = parameterize_jupyter_notebook(
|
|
||||||
COMPLETE_NOTEBOOK, {"a": 4})
|
|
||||||
|
|
||||||
self.assertEqual(pn, COMPLETE_NOTEBOOK)
|
|
||||||
|
|
||||||
pn = parameterize_jupyter_notebook(
|
|
||||||
COMPLETE_NOTEBOOK, {"s": 4})
|
|
||||||
|
|
||||||
self.assertNotEqual(pn, COMPLETE_NOTEBOOK)
|
|
||||||
self.assertEqual(
|
|
||||||
pn["cells"][0]["source"],
|
|
||||||
"# The first cell\n\ns = 4\nnum = 1000")
|
|
||||||
|
|
||||||
# Test that parameterize_python_script parameterises given script
|
|
||||||
def testParameteriseScript(self)->None:
|
|
||||||
ps = parameterize_python_script(
|
|
||||||
COMPLETE_PYTHON_SCRIPT, {})
|
|
||||||
|
|
||||||
self.assertEqual(ps, COMPLETE_PYTHON_SCRIPT)
|
|
||||||
|
|
||||||
ps = parameterize_python_script(
|
|
||||||
COMPLETE_PYTHON_SCRIPT, {"a": 50})
|
|
||||||
|
|
||||||
self.assertEqual(ps, COMPLETE_PYTHON_SCRIPT)
|
|
||||||
|
|
||||||
ps = parameterize_python_script(
|
|
||||||
COMPLETE_PYTHON_SCRIPT, {"num": 50})
|
|
||||||
|
|
||||||
self.assertNotEqual(ps, COMPLETE_PYTHON_SCRIPT)
|
|
||||||
self.assertEqual(ps[2], "num = 50")
|
|
||||||
|
|
||||||
# Test that create_event produces valid event dictionary
|
|
||||||
def testCreateEvent(self)->None:
|
|
||||||
pattern = FileEventPattern(
|
|
||||||
"pattern",
|
|
||||||
"file_path",
|
|
||||||
"recipe_one",
|
|
||||||
"infile",
|
|
||||||
parameters={
|
|
||||||
"extra":"A line from a test Pattern",
|
|
||||||
"outfile":"result_path"
|
|
||||||
})
|
|
||||||
recipe = JupyterNotebookRecipe(
|
|
||||||
"recipe_one", APPENDING_NOTEBOOK)
|
|
||||||
|
|
||||||
rule = create_rule(pattern, recipe)
|
|
||||||
|
|
||||||
event = create_event("test", "path", rule)
|
|
||||||
|
|
||||||
self.assertEqual(type(event), dict)
|
|
||||||
self.assertEqual(len(event.keys()), 3)
|
|
||||||
self.assertTrue(EVENT_TYPE in event.keys())
|
|
||||||
self.assertTrue(EVENT_PATH in event.keys())
|
|
||||||
self.assertTrue(EVENT_RULE in event.keys())
|
|
||||||
self.assertEqual(event[EVENT_TYPE], "test")
|
|
||||||
self.assertEqual(event[EVENT_PATH], "path")
|
|
||||||
self.assertEqual(event[EVENT_RULE], rule)
|
|
||||||
|
|
||||||
event2 = create_event("test2", "path2", rule, extras={"a":1})
|
|
||||||
|
|
||||||
self.assertEqual(type(event2), dict)
|
|
||||||
self.assertTrue(EVENT_TYPE in event2.keys())
|
|
||||||
self.assertTrue(EVENT_PATH in event.keys())
|
|
||||||
self.assertTrue(EVENT_RULE in event.keys())
|
|
||||||
self.assertEqual(len(event2.keys()), 4)
|
|
||||||
self.assertEqual(event2[EVENT_TYPE], "test2")
|
|
||||||
self.assertEqual(event2[EVENT_PATH], "path2")
|
|
||||||
self.assertEqual(event2[EVENT_RULE], rule)
|
|
||||||
self.assertEqual(event2["a"], 1)
|
|
||||||
|
|
||||||
# Test that create_job produces valid job dictionary
|
|
||||||
def testCreateJob(self)->None:
|
|
||||||
pattern = FileEventPattern(
|
|
||||||
"pattern",
|
|
||||||
"file_path",
|
|
||||||
"recipe_one",
|
|
||||||
"infile",
|
|
||||||
parameters={
|
|
||||||
"extra":"A line from a test Pattern",
|
|
||||||
"outfile":"result_path"
|
|
||||||
})
|
|
||||||
recipe = JupyterNotebookRecipe(
|
|
||||||
"recipe_one", APPENDING_NOTEBOOK)
|
|
||||||
|
|
||||||
rule = create_rule(pattern, recipe)
|
|
||||||
|
|
||||||
event = create_event(
|
|
||||||
EVENT_TYPE_WATCHDOG,
|
|
||||||
"file_path",
|
|
||||||
rule,
|
|
||||||
extras={
|
|
||||||
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
|
||||||
EVENT_RULE: rule,
|
|
||||||
WATCHDOG_HASH: "file_hash"
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
job_dict = create_job(
|
|
||||||
JOB_TYPE_PAPERMILL,
|
|
||||||
event,
|
|
||||||
extras={
|
|
||||||
JOB_PARAMETERS:{
|
|
||||||
"extra":"extra",
|
|
||||||
"infile":"file_path",
|
|
||||||
"outfile":"result_path"
|
|
||||||
},
|
|
||||||
JOB_HASH: "file_hash",
|
|
||||||
PYTHON_FUNC:max
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertIsInstance(job_dict, dict)
|
|
||||||
self.assertIn(JOB_ID, job_dict)
|
|
||||||
self.assertIsInstance(job_dict[JOB_ID], str)
|
|
||||||
self.assertIn(JOB_EVENT, job_dict)
|
|
||||||
self.assertEqual(job_dict[JOB_EVENT], event)
|
|
||||||
self.assertIn(JOB_TYPE, job_dict)
|
|
||||||
self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PAPERMILL)
|
|
||||||
self.assertIn(JOB_PATTERN, job_dict)
|
|
||||||
self.assertEqual(job_dict[JOB_PATTERN], pattern.name)
|
|
||||||
self.assertIn(JOB_RECIPE, job_dict)
|
|
||||||
self.assertEqual(job_dict[JOB_RECIPE], recipe.name)
|
|
||||||
self.assertIn(JOB_RULE, job_dict)
|
|
||||||
self.assertEqual(job_dict[JOB_RULE], rule.name)
|
|
||||||
self.assertIn(JOB_STATUS, job_dict)
|
|
||||||
self.assertEqual(job_dict[JOB_STATUS], STATUS_QUEUED)
|
|
||||||
self.assertIn(JOB_CREATE_TIME, job_dict)
|
|
||||||
self.assertIsInstance(job_dict[JOB_CREATE_TIME], datetime)
|
|
||||||
self.assertIn(JOB_REQUIREMENTS, job_dict)
|
|
||||||
self.assertEqual(job_dict[JOB_REQUIREMENTS], {})
|
|
||||||
|
|
||||||
# Test that replace_keywords replaces MEOW keywords in a given dictionary
|
|
||||||
def testReplaceKeywords(self)->None:
|
|
||||||
test_dict = {
|
|
||||||
"A": f"--{KEYWORD_PATH}--",
|
|
||||||
"B": f"--{KEYWORD_REL_PATH}--",
|
|
||||||
"C": f"--{KEYWORD_DIR}--",
|
|
||||||
"D": f"--{KEYWORD_REL_DIR}--",
|
|
||||||
"E": f"--{KEYWORD_FILENAME}--",
|
|
||||||
"F": f"--{KEYWORD_PREFIX}--",
|
|
||||||
"G": f"--{KEYWORD_BASE}--",
|
|
||||||
"H": f"--{KEYWORD_EXTENSION}--",
|
|
||||||
"I": f"--{KEYWORD_JOB}--",
|
|
||||||
"J": f"--{KEYWORD_PATH}-{KEYWORD_PATH}--",
|
|
||||||
"K": f"{KEYWORD_PATH}",
|
|
||||||
"L": f"--{KEYWORD_PATH}-{KEYWORD_REL_PATH}-{KEYWORD_DIR}-"
|
|
||||||
f"{KEYWORD_REL_DIR}-{KEYWORD_FILENAME}-{KEYWORD_PREFIX}-"
|
|
||||||
f"{KEYWORD_BASE}-{KEYWORD_EXTENSION}-{KEYWORD_JOB}--",
|
|
||||||
"M": "A",
|
|
||||||
"N": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
replaced = replace_keywords(
|
|
||||||
test_dict,
|
|
||||||
"job_id",
|
|
||||||
os.path.join("base", "src", "dir", "file.ext"),
|
|
||||||
os.path.join("base", "monitor", "dir")
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertIsInstance(replaced, dict)
|
|
||||||
self.assertEqual(len(test_dict.keys()), len(replaced.keys()))
|
|
||||||
for k in test_dict.keys():
|
|
||||||
self.assertIn(k, replaced)
|
|
||||||
|
|
||||||
self.assertEqual(replaced["A"],
|
|
||||||
os.path.join("--base", "src", "dir", "file.ext--"))
|
|
||||||
self.assertEqual(replaced["B"],
|
|
||||||
os.path.join("--..", "..", "src", "dir", "file.ext--"))
|
|
||||||
self.assertEqual(replaced["C"],
|
|
||||||
os.path.join("--base", "src", "dir--"))
|
|
||||||
self.assertEqual(replaced["D"],
|
|
||||||
os.path.join("--..", "..", "src", "dir--"))
|
|
||||||
self.assertEqual(replaced["E"], "--file.ext--")
|
|
||||||
self.assertEqual(replaced["F"], "--file--")
|
|
||||||
self.assertEqual(replaced["G"],
|
|
||||||
os.path.join("--base", "monitor", "dir--"))
|
|
||||||
self.assertEqual(replaced["H"], "--.ext--")
|
|
||||||
self.assertEqual(replaced["I"], "--job_id--")
|
|
||||||
self.assertEqual(replaced["J"],
|
|
||||||
os.path.join("--base", "src", "dir", "file.ext-base", "src", "dir", "file.ext--"))
|
|
||||||
self.assertEqual(replaced["K"],
|
|
||||||
os.path.join("base", "src", "dir", "file.ext"))
|
|
||||||
self.assertEqual(replaced["L"],
|
|
||||||
os.path.join("--base", "src", "dir", "file.ext-..", "..", "src", "dir", "file.ext-base", "src", "dir-"
|
|
||||||
"..", "..", "src", "dir-file.ext-file-base", "monitor", "dir-.ext-job_id--"))
|
|
||||||
self.assertEqual(replaced["M"], "A")
|
|
||||||
self.assertEqual(replaced["N"], 1)
|
|
||||||
|
|
||||||
# Test that write_file can write files
|
# Test that write_file can write files
|
||||||
def testWriteFile(self)->None:
|
def testWriteFile(self)->None:
|
||||||
@ -670,6 +300,150 @@ data"""
|
|||||||
self.assertFalse(os.path.exists(
|
self.assertFalse(os.path.exists(
|
||||||
os.path.join(TEST_MONITOR_BASE, "A", "B")))
|
os.path.join(TEST_MONITOR_BASE, "A", "B")))
|
||||||
|
|
||||||
|
# Test lines to str
|
||||||
|
def testLinesToStr(self)->None:
|
||||||
|
l = ["a", "b", "c"]
|
||||||
|
|
||||||
|
self.assertEqual(lines_to_string(l), "a\nb\nc")
|
||||||
|
|
||||||
|
|
||||||
|
class HashingTests(unittest.TestCase):
|
||||||
|
def setUp(self)->None:
|
||||||
|
super().setUp()
|
||||||
|
setup()
|
||||||
|
|
||||||
|
def tearDown(self)->None:
|
||||||
|
super().tearDown()
|
||||||
|
teardown()
|
||||||
|
|
||||||
|
# Test that get_file_hash produces the expected hash
|
||||||
|
def testGetFileHashSha256(self)->None:
|
||||||
|
file_path = os.path.join(TEST_MONITOR_BASE, "hased_file.txt")
|
||||||
|
with open(file_path, 'w') as hashed_file:
|
||||||
|
hashed_file.write("Some data\n")
|
||||||
|
expected_hash = \
|
||||||
|
"8557122088c994ba8aa5540ccbb9a3d2d8ae2887046c2db23d65f40ae63abade"
|
||||||
|
|
||||||
|
hash = get_file_hash(file_path, SHA256)
|
||||||
|
self.assertEqual(hash, expected_hash)
|
||||||
|
|
||||||
|
# Test that get_file_hash raises on a missing file
|
||||||
|
def testGetFileHashSha256NoFile(self)->None:
|
||||||
|
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
|
||||||
|
|
||||||
|
with self.assertRaises(FileNotFoundError):
|
||||||
|
get_file_hash(file_path, SHA256)
|
||||||
|
|
||||||
|
|
||||||
|
class MeowTests(unittest.TestCase):
|
||||||
|
def setUp(self)->None:
|
||||||
|
super().setUp()
|
||||||
|
setup()
|
||||||
|
|
||||||
|
def tearDown(self)->None:
|
||||||
|
super().tearDown()
|
||||||
|
teardown()
|
||||||
|
|
||||||
|
# Test that create_event produces valid event dictionary
|
||||||
|
def testCreateEvent(self)->None:
|
||||||
|
pattern = FileEventPattern(
|
||||||
|
"pattern",
|
||||||
|
"file_path",
|
||||||
|
"recipe_one",
|
||||||
|
"infile",
|
||||||
|
parameters={
|
||||||
|
"extra":"A line from a test Pattern",
|
||||||
|
"outfile":"result_path"
|
||||||
|
})
|
||||||
|
recipe = JupyterNotebookRecipe(
|
||||||
|
"recipe_one", APPENDING_NOTEBOOK)
|
||||||
|
|
||||||
|
rule = create_rule(pattern, recipe)
|
||||||
|
|
||||||
|
event = create_event("test", "path", rule)
|
||||||
|
|
||||||
|
self.assertEqual(type(event), dict)
|
||||||
|
self.assertEqual(len(event.keys()), 3)
|
||||||
|
self.assertTrue(EVENT_TYPE in event.keys())
|
||||||
|
self.assertTrue(EVENT_PATH in event.keys())
|
||||||
|
self.assertTrue(EVENT_RULE in event.keys())
|
||||||
|
self.assertEqual(event[EVENT_TYPE], "test")
|
||||||
|
self.assertEqual(event[EVENT_PATH], "path")
|
||||||
|
self.assertEqual(event[EVENT_RULE], rule)
|
||||||
|
|
||||||
|
event2 = create_event("test2", "path2", rule, extras={"a":1})
|
||||||
|
|
||||||
|
self.assertEqual(type(event2), dict)
|
||||||
|
self.assertTrue(EVENT_TYPE in event2.keys())
|
||||||
|
self.assertTrue(EVENT_PATH in event.keys())
|
||||||
|
self.assertTrue(EVENT_RULE in event.keys())
|
||||||
|
self.assertEqual(len(event2.keys()), 4)
|
||||||
|
self.assertEqual(event2[EVENT_TYPE], "test2")
|
||||||
|
self.assertEqual(event2[EVENT_PATH], "path2")
|
||||||
|
self.assertEqual(event2[EVENT_RULE], rule)
|
||||||
|
self.assertEqual(event2["a"], 1)
|
||||||
|
|
||||||
|
# Test that create_job produces valid job dictionary
|
||||||
|
def testCreateJob(self)->None:
|
||||||
|
pattern = FileEventPattern(
|
||||||
|
"pattern",
|
||||||
|
"file_path",
|
||||||
|
"recipe_one",
|
||||||
|
"infile",
|
||||||
|
parameters={
|
||||||
|
"extra":"A line from a test Pattern",
|
||||||
|
"outfile":"result_path"
|
||||||
|
})
|
||||||
|
recipe = JupyterNotebookRecipe(
|
||||||
|
"recipe_one", APPENDING_NOTEBOOK)
|
||||||
|
|
||||||
|
rule = create_rule(pattern, recipe)
|
||||||
|
|
||||||
|
event = create_event(
|
||||||
|
EVENT_TYPE_WATCHDOG,
|
||||||
|
"file_path",
|
||||||
|
rule,
|
||||||
|
extras={
|
||||||
|
WATCHDOG_BASE: TEST_MONITOR_BASE,
|
||||||
|
EVENT_RULE: rule,
|
||||||
|
WATCHDOG_HASH: "file_hash"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
job_dict = create_job(
|
||||||
|
JOB_TYPE_PAPERMILL,
|
||||||
|
event,
|
||||||
|
extras={
|
||||||
|
JOB_PARAMETERS:{
|
||||||
|
"extra":"extra",
|
||||||
|
"infile":"file_path",
|
||||||
|
"outfile":"result_path"
|
||||||
|
},
|
||||||
|
JOB_HASH: "file_hash",
|
||||||
|
PYTHON_FUNC:max
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsInstance(job_dict, dict)
|
||||||
|
self.assertIn(JOB_ID, job_dict)
|
||||||
|
self.assertIsInstance(job_dict[JOB_ID], str)
|
||||||
|
self.assertIn(JOB_EVENT, job_dict)
|
||||||
|
self.assertEqual(job_dict[JOB_EVENT], event)
|
||||||
|
self.assertIn(JOB_TYPE, job_dict)
|
||||||
|
self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PAPERMILL)
|
||||||
|
self.assertIn(JOB_PATTERN, job_dict)
|
||||||
|
self.assertEqual(job_dict[JOB_PATTERN], pattern.name)
|
||||||
|
self.assertIn(JOB_RECIPE, job_dict)
|
||||||
|
self.assertEqual(job_dict[JOB_RECIPE], recipe.name)
|
||||||
|
self.assertIn(JOB_RULE, job_dict)
|
||||||
|
self.assertEqual(job_dict[JOB_RULE], rule.name)
|
||||||
|
self.assertIn(JOB_STATUS, job_dict)
|
||||||
|
self.assertEqual(job_dict[JOB_STATUS], STATUS_QUEUED)
|
||||||
|
self.assertIn(JOB_CREATE_TIME, job_dict)
|
||||||
|
self.assertIsInstance(job_dict[JOB_CREATE_TIME], datetime)
|
||||||
|
self.assertIn(JOB_REQUIREMENTS, job_dict)
|
||||||
|
self.assertEqual(job_dict[JOB_REQUIREMENTS], {})
|
||||||
|
|
||||||
# Test creation of watchdog event dict
|
# Test creation of watchdog event dict
|
||||||
def testCreateWatchdogEvent(self)->None:
|
def testCreateWatchdogEvent(self)->None:
|
||||||
pattern = FileEventPattern(
|
pattern = FileEventPattern(
|
||||||
@ -722,8 +496,295 @@ data"""
|
|||||||
self.assertEqual(event[WATCHDOG_BASE], "base")
|
self.assertEqual(event[WATCHDOG_BASE], "base")
|
||||||
self.assertEqual(event[WATCHDOG_HASH], "hash")
|
self.assertEqual(event[WATCHDOG_HASH], "hash")
|
||||||
|
|
||||||
# Test lines to str
|
# Test that replace_keywords replaces MEOW keywords in a given dictionary
|
||||||
def testLinesToStr(self)->None:
|
def testReplaceKeywords(self)->None:
|
||||||
l = ["a", "b", "c"]
|
test_dict = {
|
||||||
|
"A": f"--{KEYWORD_PATH}--",
|
||||||
|
"B": f"--{KEYWORD_REL_PATH}--",
|
||||||
|
"C": f"--{KEYWORD_DIR}--",
|
||||||
|
"D": f"--{KEYWORD_REL_DIR}--",
|
||||||
|
"E": f"--{KEYWORD_FILENAME}--",
|
||||||
|
"F": f"--{KEYWORD_PREFIX}--",
|
||||||
|
"G": f"--{KEYWORD_BASE}--",
|
||||||
|
"H": f"--{KEYWORD_EXTENSION}--",
|
||||||
|
"I": f"--{KEYWORD_JOB}--",
|
||||||
|
"J": f"--{KEYWORD_PATH}-{KEYWORD_PATH}--",
|
||||||
|
"K": f"{KEYWORD_PATH}",
|
||||||
|
"L": f"--{KEYWORD_PATH}-{KEYWORD_REL_PATH}-{KEYWORD_DIR}-"
|
||||||
|
f"{KEYWORD_REL_DIR}-{KEYWORD_FILENAME}-{KEYWORD_PREFIX}-"
|
||||||
|
f"{KEYWORD_BASE}-{KEYWORD_EXTENSION}-{KEYWORD_JOB}--",
|
||||||
|
"M": "A",
|
||||||
|
"N": 1
|
||||||
|
}
|
||||||
|
|
||||||
self.assertEqual(lines_to_string(l), "a\nb\nc")
|
replaced = replace_keywords(
|
||||||
|
test_dict,
|
||||||
|
"job_id",
|
||||||
|
os.path.join("base", "src", "dir", "file.ext"),
|
||||||
|
os.path.join("base", "monitor", "dir")
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsInstance(replaced, dict)
|
||||||
|
self.assertEqual(len(test_dict.keys()), len(replaced.keys()))
|
||||||
|
for k in test_dict.keys():
|
||||||
|
self.assertIn(k, replaced)
|
||||||
|
|
||||||
|
self.assertEqual(replaced["A"],
|
||||||
|
os.path.join("--base", "src", "dir", "file.ext--"))
|
||||||
|
self.assertEqual(replaced["B"],
|
||||||
|
os.path.join("--..", "..", "src", "dir", "file.ext--"))
|
||||||
|
self.assertEqual(replaced["C"],
|
||||||
|
os.path.join("--base", "src", "dir--"))
|
||||||
|
self.assertEqual(replaced["D"],
|
||||||
|
os.path.join("--..", "..", "src", "dir--"))
|
||||||
|
self.assertEqual(replaced["E"], "--file.ext--")
|
||||||
|
self.assertEqual(replaced["F"], "--file--")
|
||||||
|
self.assertEqual(replaced["G"],
|
||||||
|
os.path.join("--base", "monitor", "dir--"))
|
||||||
|
self.assertEqual(replaced["H"], "--.ext--")
|
||||||
|
self.assertEqual(replaced["I"], "--job_id--")
|
||||||
|
self.assertEqual(replaced["J"],
|
||||||
|
os.path.join("--base", "src", "dir", "file.ext-base", "src", "dir", "file.ext--"))
|
||||||
|
self.assertEqual(replaced["K"],
|
||||||
|
os.path.join("base", "src", "dir", "file.ext"))
|
||||||
|
self.assertEqual(replaced["L"],
|
||||||
|
os.path.join("--base", "src", "dir", "file.ext-..", "..", "src", "dir", "file.ext-base", "src", "dir-"
|
||||||
|
"..", "..", "src", "dir-file.ext-file-base", "monitor", "dir-.ext-job_id--"))
|
||||||
|
self.assertEqual(replaced["M"], "A")
|
||||||
|
self.assertEqual(replaced["N"], 1)
|
||||||
|
|
||||||
|
|
||||||
|
class NamingTests(unittest.TestCase):
|
||||||
|
def setUp(self)->None:
|
||||||
|
super().setUp()
|
||||||
|
setup()
|
||||||
|
|
||||||
|
def tearDown(self)->None:
|
||||||
|
super().tearDown()
|
||||||
|
teardown()
|
||||||
|
|
||||||
|
# Test that generate_id creates unique ids
|
||||||
|
def testGenerateIDWorking(self)->None:
|
||||||
|
id = generate_id()
|
||||||
|
self.assertEqual(len(id), 16)
|
||||||
|
for i in range(len(id)):
|
||||||
|
self.assertIn(id[i], CHAR_UPPERCASE+CHAR_LOWERCASE)
|
||||||
|
|
||||||
|
# In extrememly rare cases this may fail due to randomness in algorithm
|
||||||
|
new_id = generate_id(existing_ids=[id])
|
||||||
|
self.assertNotEqual(id, new_id)
|
||||||
|
|
||||||
|
another_id = generate_id(length=32)
|
||||||
|
self.assertEqual(len(another_id), 32)
|
||||||
|
|
||||||
|
again_id = generate_id(charset="a")
|
||||||
|
for i in range(len(again_id)):
|
||||||
|
self.assertIn(again_id[i], "a")
|
||||||
|
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
generate_id(length=2, charset="a", existing_ids=["aa"])
|
||||||
|
|
||||||
|
prefix_id = generate_id(length=4, prefix="Test")
|
||||||
|
self.assertEqual(prefix_id, "Test")
|
||||||
|
|
||||||
|
prefix_id = generate_id(prefix="Test")
|
||||||
|
self.assertEqual(len(prefix_id), 16)
|
||||||
|
self.assertTrue(prefix_id.startswith("Test"))
|
||||||
|
|
||||||
|
|
||||||
|
class ParameterisationTests(unittest.TestCase):
|
||||||
|
def setUp(self)->None:
|
||||||
|
super().setUp()
|
||||||
|
setup()
|
||||||
|
|
||||||
|
def tearDown(self)->None:
|
||||||
|
super().tearDown()
|
||||||
|
teardown()
|
||||||
|
|
||||||
|
# Test that parameterize_jupyter_notebook parameterises given notebook
|
||||||
|
def testParameteriseNotebook(self)->None:
|
||||||
|
pn = parameterize_jupyter_notebook(
|
||||||
|
COMPLETE_NOTEBOOK, {})
|
||||||
|
|
||||||
|
self.assertEqual(pn, COMPLETE_NOTEBOOK)
|
||||||
|
|
||||||
|
pn = parameterize_jupyter_notebook(
|
||||||
|
COMPLETE_NOTEBOOK, {"a": 4})
|
||||||
|
|
||||||
|
self.assertEqual(pn, COMPLETE_NOTEBOOK)
|
||||||
|
|
||||||
|
pn = parameterize_jupyter_notebook(
|
||||||
|
COMPLETE_NOTEBOOK, {"s": 4})
|
||||||
|
|
||||||
|
self.assertNotEqual(pn, COMPLETE_NOTEBOOK)
|
||||||
|
self.assertEqual(
|
||||||
|
pn["cells"][0]["source"],
|
||||||
|
"# The first cell\n\ns = 4\nnum = 1000")
|
||||||
|
|
||||||
|
# Test that parameterize_python_script parameterises given script
|
||||||
|
def testParameteriseScript(self)->None:
|
||||||
|
ps = parameterize_python_script(
|
||||||
|
COMPLETE_PYTHON_SCRIPT, {})
|
||||||
|
|
||||||
|
self.assertEqual(ps, COMPLETE_PYTHON_SCRIPT)
|
||||||
|
|
||||||
|
ps = parameterize_python_script(
|
||||||
|
COMPLETE_PYTHON_SCRIPT, {"a": 50})
|
||||||
|
|
||||||
|
self.assertEqual(ps, COMPLETE_PYTHON_SCRIPT)
|
||||||
|
|
||||||
|
ps = parameterize_python_script(
|
||||||
|
COMPLETE_PYTHON_SCRIPT, {"num": 50})
|
||||||
|
|
||||||
|
self.assertNotEqual(ps, COMPLETE_PYTHON_SCRIPT)
|
||||||
|
self.assertEqual(ps[2], "num = 50")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessIoTests(unittest.TestCase):
|
||||||
|
def setUp(self)->None:
|
||||||
|
super().setUp()
|
||||||
|
setup()
|
||||||
|
|
||||||
|
def tearDown(self)->None:
|
||||||
|
super().tearDown()
|
||||||
|
teardown()
|
||||||
|
|
||||||
|
# Test that wait can wait on multiple pipes
|
||||||
|
def testWaitPipes(self)->None:
|
||||||
|
pipe_one_reader, pipe_one_writer = Pipe()
|
||||||
|
pipe_two_reader, pipe_two_writer = Pipe()
|
||||||
|
|
||||||
|
inputs = [
|
||||||
|
pipe_one_reader, pipe_two_reader
|
||||||
|
]
|
||||||
|
|
||||||
|
pipe_one_writer.send(1)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(pipe_one_reader, readables)
|
||||||
|
self.assertEqual(len(readables), 1)
|
||||||
|
msg = readables[0].recv()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
|
||||||
|
pipe_one_writer.send(1)
|
||||||
|
pipe_two_writer.send(2)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(pipe_one_reader, readables)
|
||||||
|
self.assertIn(pipe_two_reader, readables)
|
||||||
|
self.assertEqual(len(readables), 2)
|
||||||
|
for readable in readables:
|
||||||
|
if readable == pipe_one_reader:
|
||||||
|
msg = readable.recv()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
elif readable == pipe_two_reader:
|
||||||
|
msg = readable.recv()
|
||||||
|
self.assertEqual(msg, 2)
|
||||||
|
|
||||||
|
# Test that wait can wait on multiple queues
|
||||||
|
def testWaitQueues(self)->None:
|
||||||
|
queue_one = Queue()
|
||||||
|
queue_two = Queue()
|
||||||
|
|
||||||
|
inputs = [
|
||||||
|
queue_one, queue_two
|
||||||
|
]
|
||||||
|
|
||||||
|
queue_one.put(1)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(queue_one, readables)
|
||||||
|
self.assertEqual(len(readables), 1)
|
||||||
|
msg = readables[0].get()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
|
||||||
|
queue_one.put(1)
|
||||||
|
queue_two.put(2)
|
||||||
|
sleep(0.1)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(queue_one, readables)
|
||||||
|
self.assertIn(queue_two, readables)
|
||||||
|
self.assertEqual(len(readables), 2)
|
||||||
|
for readable in readables:
|
||||||
|
if readable == queue_one:
|
||||||
|
msg = readable.get()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
elif readable == queue_two:
|
||||||
|
msg = readable.get()
|
||||||
|
self.assertEqual(msg, 2)
|
||||||
|
|
||||||
|
# Test that wait can wait on multiple pipes and queues
|
||||||
|
def testWaitPipesAndQueues(self)->None:
|
||||||
|
pipe_one_reader, pipe_one_writer = Pipe()
|
||||||
|
pipe_two_reader, pipe_two_writer = Pipe()
|
||||||
|
queue_one = Queue()
|
||||||
|
queue_two = Queue()
|
||||||
|
|
||||||
|
inputs = [
|
||||||
|
pipe_one_reader, pipe_two_reader, queue_one, queue_two
|
||||||
|
]
|
||||||
|
|
||||||
|
pipe_one_writer.send(1)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(pipe_one_reader, readables)
|
||||||
|
self.assertEqual(len(readables), 1)
|
||||||
|
msg = readables[0].recv()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
|
||||||
|
pipe_one_writer.send(1)
|
||||||
|
pipe_two_writer.send(2)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(pipe_one_reader, readables)
|
||||||
|
self.assertIn(pipe_two_reader, readables)
|
||||||
|
self.assertEqual(len(readables), 2)
|
||||||
|
for readable in readables:
|
||||||
|
if readable == pipe_one_reader:
|
||||||
|
msg = readable.recv()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
if readable == pipe_two_reader:
|
||||||
|
msg = readable.recv()
|
||||||
|
self.assertEqual(msg, 2)
|
||||||
|
|
||||||
|
queue_one.put(1)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(queue_one, readables)
|
||||||
|
self.assertEqual(len(readables), 1)
|
||||||
|
msg = readables[0].get()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
|
||||||
|
queue_one.put(1)
|
||||||
|
queue_two.put(2)
|
||||||
|
sleep(0.1)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(queue_one, readables)
|
||||||
|
self.assertIn(queue_two, readables)
|
||||||
|
self.assertEqual(len(readables), 2)
|
||||||
|
for readable in readables:
|
||||||
|
if readable == queue_one:
|
||||||
|
msg = readable.get()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
elif readable == queue_two:
|
||||||
|
msg = readable.get()
|
||||||
|
self.assertEqual(msg, 2)
|
||||||
|
|
||||||
|
queue_one.put(1)
|
||||||
|
pipe_one_writer.send(1)
|
||||||
|
sleep(0.1)
|
||||||
|
readables = wait(inputs)
|
||||||
|
|
||||||
|
self.assertIn(queue_one, readables)
|
||||||
|
self.assertIn(pipe_one_reader, readables)
|
||||||
|
self.assertEqual(len(readables), 2)
|
||||||
|
for readable in readables:
|
||||||
|
if readable == queue_one:
|
||||||
|
msg = readable.get()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
elif readable == pipe_one_reader:
|
||||||
|
msg = readable.recv()
|
||||||
|
self.assertEqual(msg, 1)
|
||||||
|
|||||||
Reference in New Issue
Block a user