NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added test for complete execution

Browse Source
This commit is contained in:
PatchOfScotland
2023-01-10 13:34:41 +01:00
parent ea9a689b26
commit 5dded8fc96
6 changed files with 445 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_all.sh
View File

@ -13,7 +13,7 @@ for entry in "$search_dir"/*
do
if [[ $entry == ./test* ]] && [[ $entry != ./$script_name ]];
then
pytest $entry
pytest $entry "-W ignore::DeprecationWarning"
fi
done

49
tests/test_functionality.py
View File

@ -1,13 +1,15 @@
import unittest
import os
from multiprocessing import Pipe, Queue
from time import sleep
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
BAREBONES_NOTEBOOK
BAREBONES_NOTEBOOK, SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK
from core.functionality import create_rules, generate_id, wait, \
check_pattern_dict, check_recipe_dict
check_pattern_dict, check_recipe_dict, get_file_hash, rmtree, make_dir, \
parameterize_jupyter_notebook
from core.meow import BaseRule
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
@ -22,12 +24,16 @@ valid_recipe_one = JupyterNotebookRecipe(
valid_recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
return super().setUp()
super().setUp()
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
def tearDown(self) -> None:
return super().tearDown()
super().tearDown()
rmtree(TEST_MONITOR_BASE)
def testCreateRulesMinimum(self)->None:
create_rules({}, {})
@ -213,7 +219,6 @@ class CorrectnessTests(unittest.TestCase):
msg = readable.get()
self.assertEqual(msg, 2)
def testWaitPipesAndQueues(self)->None:
pipe_one_reader, pipe_one_writer = Pipe()
pipe_two_reader, pipe_two_writer = Pipe()
@ -287,3 +292,37 @@ class CorrectnessTests(unittest.TestCase):
msg = readable.recv()
self.assertEqual(msg, 1)
def testGetFileHashSha256(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "hased_file.txt")
with open(file_path, 'w') as hashed_file:
hashed_file.write("Some data\n")
expected_hash = \
"8557122088c994ba8aa5540ccbb9a3d2d8ae2887046c2db23d65f40ae63abade"
hash = get_file_hash(file_path, SHA256)
self.assertEqual(hash, expected_hash)
def testGetFileHashSha256NoFile(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
with self.assertRaises(FileNotFoundError):
get_file_hash(file_path, SHA256)
def testParameteriseNotebook(self)->None:
pn = parameterize_jupyter_notebook(
COMPLETE_NOTEBOOK, {})
self.assertEqual(pn, COMPLETE_NOTEBOOK)
pn = parameterize_jupyter_notebook(
COMPLETE_NOTEBOOK, {"a": 4})
self.assertEqual(pn, COMPLETE_NOTEBOOK)
pn = parameterize_jupyter_notebook(
COMPLETE_NOTEBOOK, {"s": 4})
self.assertNotEqual(pn, COMPLETE_NOTEBOOK)
self.assertEqual(
pn["cells"][0]["source"],
"# The first cell\n\ns = 4\nnum = 1000")

227
tests/test_meow.py
View File

@ -1,19 +1,34 @@
import io
import os
import unittest
from multiprocessing import Pipe
from time import sleep
from typing import Any
from core.correctness.vars import BAREBONES_NOTEBOOK
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
TEST_MONITOR_BASE, APPENDING_NOTEBOOK
from core.functionality import make_dir, rmtree, create_rules, read_notebook
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler
BaseHandler, MeowRunner
from patterns import WatchdogMonitor, FileEventPattern
from recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe, RESULT_FILE
class MeowTests(unittest.TestCase):
def setUp(self)->None:
return super().setUp()
def tearDown(self)->None:
return super().tearDown()
def setUp(self) -> None:
super().setUp()
make_dir(TEST_MONITOR_BASE)
make_dir(TEST_HANDLER_BASE)
make_dir(TEST_JOB_OUTPUT)
def tearDown(self) -> None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree(TEST_HANDLER_BASE)
rmtree(TEST_JOB_OUTPUT)
def testBaseRecipe(self)->None:
with self.assertRaises(TypeError):
@ -111,3 +126,201 @@ class MeowTests(unittest.TestCase):
pass
FullTestHandler("")
def testMeowRunner(self)->None:
monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"{VGRID}/output/{FILENAME}"
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
monitor_debug_stream = io.StringIO("")
handler_debug_stream = io.StringIO("")
runner = MeowRunner(
WatchdogMonitor(
TEST_MONITOR_BASE,
rules,
monitor_to_handler_writer,
print=monitor_debug_stream,
logging=3, settletime=1
),
PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=handler_debug_stream,
logging=3
)
)
runner.start()
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
loops = 0
job_id = None
while loops < 15:
sleep(1)
handler_debug_stream.seek(0)
messages = handler_debug_stream.readlines()
for msg in messages:
self.assertNotIn("ERROR", msg)
if "INFO: Completed job " in msg:
job_id = msg.replace("INFO: Completed job ", "")
job_id = job_id[:job_id.index(" with output")]
loops = 15
loops += 1
self.assertIsNotNone(job_id)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
runner.stop()
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(job_dir)), 5)
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
self.assertIsNotNone(result)
output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
self.assertTrue(os.path.exists(output_path))
with open(output_path, "r") as f:
data = f.read()
self.assertEqual(data, "Initial Data\nA line from a test Pattern")
def testMeowRunnerLinkeExecution(self)->None:
monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
parameters={
"extra":"A line from Pattern 1",
"outfile":"{VGRID}/middle/{FILENAME}"
})
pattern_two = FileEventPattern(
"pattern_two", "middle/A.txt", "recipe_one", "infile",
parameters={
"extra":"A line from Pattern 2",
"outfile":"{VGRID}/output/{FILENAME}"
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
pattern_two.name: pattern_two,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
monitor_debug_stream = io.StringIO("")
handler_debug_stream = io.StringIO("")
runner = MeowRunner(
WatchdogMonitor(
TEST_MONITOR_BASE,
rules,
monitor_to_handler_writer,
print=monitor_debug_stream,
logging=3, settletime=1
),
PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=handler_debug_stream,
logging=3
)
)
runner.start()
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
make_dir(start_dir)
self.assertTrue(start_dir)
with open(os.path.join(start_dir, "A.txt"), "w") as f:
f.write("Initial Data")
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
loops = 0
job_ids = []
while len(job_ids) < 2 or loops < 30:
sleep(1)
handler_debug_stream.seek(0)
messages = handler_debug_stream.readlines()
for msg in messages:
self.assertNotIn("ERROR", msg)
if "INFO: Completed job " in msg:
job_id = msg.replace("INFO: Completed job ", "")
job_id = job_id[:job_id.index(" with output")]
if job_id not in job_ids:
job_ids.append(job_id)
loops += 1
print(job_ids)
self.assertEqual(len(job_ids), 2)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 2)
self.assertIn(job_ids[0], os.listdir(TEST_JOB_OUTPUT))
self.assertIn(job_ids[1], os.listdir(TEST_JOB_OUTPUT))
runner.stop()
mid_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(mid_job_dir)), 5)
result = read_notebook(os.path.join(mid_job_dir, RESULT_FILE))
self.assertIsNotNone(result)
mid_output_path = os.path.join(TEST_MONITOR_BASE, "middle", "A.txt")
self.assertTrue(os.path.exists(mid_output_path))
with open(mid_output_path, "r") as f:
data = f.read()
self.assertEqual(data, "Initial Data\nA line from Pattern 1")
final_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(final_job_dir)), 5)
result = read_notebook(os.path.join(final_job_dir, RESULT_FILE))
self.assertIsNotNone(result)
final_output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
self.assertTrue(os.path.exists(final_output_path))
with open(final_output_path, "r") as f:
data = f.read()
self.assertEqual(data,
"Initial Data\nA line from Pattern 1\nA line from Pattern 2")

32
tests/test_patterns.py
View File

@ -1,28 +1,24 @@
import shutil
import os
import unittest
from multiprocessing import Pipe
from core.correctness.vars import FILE_EVENTS, FILE_CREATE_EVENT, PIPE_READ, \
PIPE_WRITE, BAREBONES_NOTEBOOK
from core.functionality import create_rules
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor
from core.correctness.vars import FILE_EVENTS, FILE_CREATE_EVENT, \
BAREBONES_NOTEBOOK, TEST_MONITOR_BASE
from core.functionality import create_rules, rmtree, make_dir
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
_DEFAULT_MASK
from recipes import JupyterNotebookRecipe
TEST_BASE = "test_base"
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
super().setUp()
if not os.path.exists(TEST_BASE):
os.mkdir(TEST_BASE)
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
def tearDown(self) -> None:
super().tearDown()
if os.path.exists(TEST_BASE):
shutil.rmtree(TEST_BASE)
rmtree(TEST_MONITOR_BASE)
def testFileEventPatternCreationMinimum(self)->None:
FileEventPattern("name", "path", "recipe", "file")
@ -95,7 +91,7 @@ class CorrectnessTests(unittest.TestCase):
def testFileEventPatternEventMask(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
self.assertEqual(fep.event_mask, FILE_EVENTS)
self.assertEqual(fep.event_mask, _DEFAULT_MASK)
with self.assertRaises(TypeError):
fep = FileEventPattern("name", "path", "recipe", "file",
@ -109,11 +105,9 @@ class CorrectnessTests(unittest.TestCase):
fep = FileEventPattern("name", "path", "recipe", "file",
event_mask=[FILE_CREATE_EVENT, "nope"])
self.assertEqual(fep.event_mask, FILE_EVENTS)
def testWatchdogMonitorMinimum(self)->None:
from_monitor = Pipe()
WatchdogMonitor(TEST_BASE, {}, from_monitor[PIPE_WRITE])
WatchdogMonitor(TEST_MONITOR_BASE, {}, from_monitor[1])
def testWatchdogMonitorEventIdentificaion(self)->None:
from_monitor_reader, from_monitor_writer = Pipe()
@ -131,11 +125,11 @@ class CorrectnessTests(unittest.TestCase):
}
rules = create_rules(patterns, recipes)
wm = WatchdogMonitor(TEST_BASE, rules, from_monitor_writer)
wm = WatchdogMonitor(TEST_MONITOR_BASE, rules, from_monitor_writer)
wm.start()
open(os.path.join(TEST_BASE, "A"), "w")
open(os.path.join(TEST_MONITOR_BASE, "A"), "w")
if from_monitor_reader.poll(3):
message = from_monitor_reader.recv()
@ -143,9 +137,9 @@ class CorrectnessTests(unittest.TestCase):
event, rule = message
self.assertIsNotNone(event)
self.assertIsNotNone(rule)
self.assertEqual(event.src_path, os.path.join(TEST_BASE, "A"))
self.assertEqual(event.src_path, os.path.join(TEST_MONITOR_BASE, "A"))
open(os.path.join(TEST_BASE, "B"), "w")
open(os.path.join(TEST_MONITOR_BASE, "B"), "w")
if from_monitor_reader.poll(3):
new_message = from_monitor_reader.recv()
else:

124
tests/test_recipes.py
View File

@ -1,19 +1,33 @@
import io
import jsonschema
import os
import unittest
from multiprocessing import Pipe
from time import sleep
from watchdog.events import FileCreatedEvent
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler
from core.correctness.vars import BAREBONES_NOTEBOOK
PapermillHandler, BASE_FILE, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from core.correctness.vars import BAREBONES_NOTEBOOK, TEST_HANDLER_BASE, \
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK
from core.functionality import rmtree, make_dir, create_rules, read_notebook
class CorrectnessTests(unittest.TestCase):
def setUp(self)->None:
return super().setUp()
def setUp(self) -> None:
super().setUp()
make_dir(TEST_MONITOR_BASE)
make_dir(TEST_HANDLER_BASE)
make_dir(TEST_JOB_OUTPUT)
def tearDown(self)->None:
return super().tearDown()
def tearDown(self) -> None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
rmtree(TEST_HANDLER_BASE)
rmtree(TEST_JOB_OUTPUT)
def testJupyterNotebookRecipeCreationMinimum(self)->None:
JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
@ -80,12 +94,20 @@ class CorrectnessTests(unittest.TestCase):
def testPapermillHanderMinimum(self)->None:
monitor_to_handler_reader, _ = Pipe()
PapermillHandler([monitor_to_handler_reader])
PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
def testPapermillHanderStartStop(self)->None:
monitor_to_handler_reader, _ = Pipe()
ph = PapermillHandler([monitor_to_handler_reader])
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.start()
ph.stop()
@ -93,7 +115,11 @@ class CorrectnessTests(unittest.TestCase):
def testPapermillHanderRepeatedStarts(self)->None:
monitor_to_handler_reader, _ = Pipe()
ph = PapermillHandler([monitor_to_handler_reader])
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.start()
with self.assertRaises(RuntimeWarning):
@ -103,9 +129,87 @@ class CorrectnessTests(unittest.TestCase):
def testPapermillHanderStopBeforeStart(self)->None:
monitor_to_handler_reader, _ = Pipe()
ph = PapermillHandler([monitor_to_handler_reader])
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
with self.assertRaises(RuntimeWarning):
ph.stop()
def testPapermillHandlerHandling(self)->None:
monitor_to_handler_reader, to_handler = Pipe()
debug_stream = io.StringIO("")
ph = PapermillHandler(
[monitor_to_handler_reader],
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT,
print=debug_stream,
logging=3
)
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
event = FileCreatedEvent(os.path.join(TEST_MONITOR_BASE, "A"))
event.monitor_base = TEST_MONITOR_BASE
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = JupyterNotebookRecipe(
"recipe_one", COMPLETE_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventJupyterNotebookRule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
ph.start()
to_handler.send((event, rule))
loops = 0
job_id = None
while loops < 15:
sleep(1)
debug_stream.seek(0)
messages = debug_stream.readlines()
for msg in messages:
self.assertNotIn("ERROR", msg)
if "INFO: Completed job " in msg:
job_id = msg.replace("INFO: Completed job ", "")
job_id = job_id[:job_id.index(" with output")]
loops = 15
loops += 1
self.assertIsNotNone(job_id)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(job_dir)), 5)
self.assertIn(META_FILE, os.listdir(job_dir))
self.assertIn(BASE_FILE, os.listdir(job_dir))
self.assertIn(PARAMS_FILE, os.listdir(job_dir))
self.assertIn(JOB_FILE, os.listdir(job_dir))
self.assertIn(RESULT_FILE, os.listdir(job_dir))
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
self.assertEqual("124875.0\n",
result["cells"][4]["outputs"][0]["text"][0])
ph.stop()

60
tests/test_validation.py
View File

@ -1,19 +1,23 @@
import unittest
import os
from typing import Any, Union
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list
from core.correctness.vars import VALID_NAME_CHARS
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_existing_dir_path, valid_non_existing_path
from core.correctness.vars import VALID_NAME_CHARS, TEST_MONITOR_BASE, SHA256
from core.functionality import rmtree, make_dir
class CorrectnessTests(unittest.TestCase):
def setUp(self)->None:
return super().setUp()
def setUp(self) -> None:
super().setUp()
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
def tearDown(self)->None:
return super().tearDown()
def tearDown(self) -> None:
super().tearDown()
rmtree(TEST_MONITOR_BASE)
def testCheckTypeValid(self)->None:
check_type(1, int)
@ -158,3 +162,45 @@ class CorrectnessTests(unittest.TestCase):
pass
check_implementation(Child.func, Parent)
def testValidExistingFilePath(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
with open(file_path, 'w') as hashed_file:
hashed_file.write("Some data\n")
valid_existing_file_path(file_path)
with self.assertRaises(FileNotFoundError):
valid_existing_file_path("not_existing_"+file_path, SHA256)
dir_path = os.path.join(TEST_MONITOR_BASE, "dir")
make_dir(dir_path)
with self.assertRaises(ValueError):
valid_existing_file_path(dir_path, SHA256)
def testValidExistingDirPath(self)->None:
valid_existing_dir_path(TEST_MONITOR_BASE)
dir_path = os.path.join(TEST_MONITOR_BASE, "dir")
with self.assertRaises(FileNotFoundError):
valid_existing_dir_path("not_existing_"+dir_path, SHA256)
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
with open(file_path, 'w') as hashed_file:
hashed_file.write("Some data\n")
with self.assertRaises(ValueError):
valid_existing_dir_path(file_path, SHA256)
def testValidNonExistingPath(self)->None:
valid_non_existing_path("does_not_exist")
make_dir("first")
with self.assertRaises(ValueError):
valid_non_existing_path("first")
make_dir("first/second")
with self.assertRaises(ValueError):
valid_non_existing_path("first/second")