added test for complete execution
This commit is contained in:
@ -13,7 +13,7 @@ for entry in "$search_dir"/*
|
|||||||
do
|
do
|
||||||
if [[ $entry == ./test* ]] && [[ $entry != ./$script_name ]];
|
if [[ $entry == ./test* ]] && [[ $entry != ./$script_name ]];
|
||||||
then
|
then
|
||||||
pytest $entry
|
pytest $entry "-W ignore::DeprecationWarning"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
@ -1,13 +1,15 @@
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
import os
|
||||||
|
|
||||||
from multiprocessing import Pipe, Queue
|
from multiprocessing import Pipe, Queue
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
||||||
BAREBONES_NOTEBOOK
|
BAREBONES_NOTEBOOK, SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK
|
||||||
from core.functionality import create_rules, generate_id, wait, \
|
from core.functionality import create_rules, generate_id, wait, \
|
||||||
check_pattern_dict, check_recipe_dict
|
check_pattern_dict, check_recipe_dict, get_file_hash, rmtree, make_dir, \
|
||||||
|
parameterize_jupyter_notebook
|
||||||
from core.meow import BaseRule
|
from core.meow import BaseRule
|
||||||
from patterns.file_event_pattern import FileEventPattern
|
from patterns.file_event_pattern import FileEventPattern
|
||||||
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
|
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
|
||||||
@ -22,12 +24,16 @@ valid_recipe_one = JupyterNotebookRecipe(
|
|||||||
valid_recipe_two = JupyterNotebookRecipe(
|
valid_recipe_two = JupyterNotebookRecipe(
|
||||||
"recipe_two", BAREBONES_NOTEBOOK)
|
"recipe_two", BAREBONES_NOTEBOOK)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
return super().setUp()
|
super().setUp()
|
||||||
|
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
def tearDown(self) -> None:
|
||||||
return super().tearDown()
|
super().tearDown()
|
||||||
|
rmtree(TEST_MONITOR_BASE)
|
||||||
|
|
||||||
def testCreateRulesMinimum(self)->None:
|
def testCreateRulesMinimum(self)->None:
|
||||||
create_rules({}, {})
|
create_rules({}, {})
|
||||||
@ -213,7 +219,6 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
msg = readable.get()
|
msg = readable.get()
|
||||||
self.assertEqual(msg, 2)
|
self.assertEqual(msg, 2)
|
||||||
|
|
||||||
|
|
||||||
def testWaitPipesAndQueues(self)->None:
|
def testWaitPipesAndQueues(self)->None:
|
||||||
pipe_one_reader, pipe_one_writer = Pipe()
|
pipe_one_reader, pipe_one_writer = Pipe()
|
||||||
pipe_two_reader, pipe_two_writer = Pipe()
|
pipe_two_reader, pipe_two_writer = Pipe()
|
||||||
@ -287,3 +292,37 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
msg = readable.recv()
|
msg = readable.recv()
|
||||||
self.assertEqual(msg, 1)
|
self.assertEqual(msg, 1)
|
||||||
|
|
||||||
|
def testGetFileHashSha256(self)->None:
|
||||||
|
file_path = os.path.join(TEST_MONITOR_BASE, "hased_file.txt")
|
||||||
|
with open(file_path, 'w') as hashed_file:
|
||||||
|
hashed_file.write("Some data\n")
|
||||||
|
expected_hash = \
|
||||||
|
"8557122088c994ba8aa5540ccbb9a3d2d8ae2887046c2db23d65f40ae63abade"
|
||||||
|
|
||||||
|
hash = get_file_hash(file_path, SHA256)
|
||||||
|
self.assertEqual(hash, expected_hash)
|
||||||
|
|
||||||
|
def testGetFileHashSha256NoFile(self)->None:
|
||||||
|
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
|
||||||
|
|
||||||
|
with self.assertRaises(FileNotFoundError):
|
||||||
|
get_file_hash(file_path, SHA256)
|
||||||
|
|
||||||
|
def testParameteriseNotebook(self)->None:
|
||||||
|
pn = parameterize_jupyter_notebook(
|
||||||
|
COMPLETE_NOTEBOOK, {})
|
||||||
|
|
||||||
|
self.assertEqual(pn, COMPLETE_NOTEBOOK)
|
||||||
|
|
||||||
|
pn = parameterize_jupyter_notebook(
|
||||||
|
COMPLETE_NOTEBOOK, {"a": 4})
|
||||||
|
|
||||||
|
self.assertEqual(pn, COMPLETE_NOTEBOOK)
|
||||||
|
|
||||||
|
pn = parameterize_jupyter_notebook(
|
||||||
|
COMPLETE_NOTEBOOK, {"s": 4})
|
||||||
|
|
||||||
|
self.assertNotEqual(pn, COMPLETE_NOTEBOOK)
|
||||||
|
self.assertEqual(
|
||||||
|
pn["cells"][0]["source"],
|
||||||
|
"# The first cell\n\ns = 4\nnum = 1000")
|
||||||
|
@ -1,19 +1,34 @@
|
|||||||
|
|
||||||
|
import io
|
||||||
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
|
from multiprocessing import Pipe
|
||||||
|
from time import sleep
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from core.correctness.vars import BAREBONES_NOTEBOOK
|
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
|
||||||
|
TEST_MONITOR_BASE, APPENDING_NOTEBOOK
|
||||||
|
from core.functionality import make_dir, rmtree, create_rules, read_notebook
|
||||||
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
|
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
|
||||||
BaseHandler
|
BaseHandler, MeowRunner
|
||||||
|
from patterns import WatchdogMonitor, FileEventPattern
|
||||||
|
from recipes.jupyter_notebook_recipe import PapermillHandler, \
|
||||||
|
JupyterNotebookRecipe, RESULT_FILE
|
||||||
|
|
||||||
|
|
||||||
class MeowTests(unittest.TestCase):
|
class MeowTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self) -> None:
|
||||||
return super().setUp()
|
super().setUp()
|
||||||
|
make_dir(TEST_MONITOR_BASE)
|
||||||
def tearDown(self)->None:
|
make_dir(TEST_HANDLER_BASE)
|
||||||
return super().tearDown()
|
make_dir(TEST_JOB_OUTPUT)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
super().tearDown()
|
||||||
|
rmtree(TEST_MONITOR_BASE)
|
||||||
|
rmtree(TEST_HANDLER_BASE)
|
||||||
|
rmtree(TEST_JOB_OUTPUT)
|
||||||
|
|
||||||
def testBaseRecipe(self)->None:
|
def testBaseRecipe(self)->None:
|
||||||
with self.assertRaises(TypeError):
|
with self.assertRaises(TypeError):
|
||||||
@ -111,3 +126,201 @@ class MeowTests(unittest.TestCase):
|
|||||||
pass
|
pass
|
||||||
FullTestHandler("")
|
FullTestHandler("")
|
||||||
|
|
||||||
|
def testMeowRunner(self)->None:
|
||||||
|
monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
|
||||||
|
|
||||||
|
pattern_one = FileEventPattern(
|
||||||
|
"pattern_one", "start/A.txt", "recipe_one", "infile",
|
||||||
|
parameters={
|
||||||
|
"extra":"A line from a test Pattern",
|
||||||
|
"outfile":"{VGRID}/output/{FILENAME}"
|
||||||
|
})
|
||||||
|
recipe = JupyterNotebookRecipe(
|
||||||
|
"recipe_one", APPENDING_NOTEBOOK)
|
||||||
|
|
||||||
|
patterns = {
|
||||||
|
pattern_one.name: pattern_one,
|
||||||
|
}
|
||||||
|
recipes = {
|
||||||
|
recipe.name: recipe,
|
||||||
|
}
|
||||||
|
rules = create_rules(patterns, recipes)
|
||||||
|
|
||||||
|
monitor_debug_stream = io.StringIO("")
|
||||||
|
handler_debug_stream = io.StringIO("")
|
||||||
|
|
||||||
|
runner = MeowRunner(
|
||||||
|
WatchdogMonitor(
|
||||||
|
TEST_MONITOR_BASE,
|
||||||
|
rules,
|
||||||
|
monitor_to_handler_writer,
|
||||||
|
print=monitor_debug_stream,
|
||||||
|
logging=3, settletime=1
|
||||||
|
),
|
||||||
|
PapermillHandler(
|
||||||
|
[monitor_to_handler_reader],
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT,
|
||||||
|
print=handler_debug_stream,
|
||||||
|
logging=3
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
runner.start()
|
||||||
|
|
||||||
|
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
|
||||||
|
make_dir(start_dir)
|
||||||
|
self.assertTrue(start_dir)
|
||||||
|
with open(os.path.join(start_dir, "A.txt"), "w") as f:
|
||||||
|
f.write("Initial Data")
|
||||||
|
|
||||||
|
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
|
||||||
|
|
||||||
|
loops = 0
|
||||||
|
job_id = None
|
||||||
|
while loops < 15:
|
||||||
|
sleep(1)
|
||||||
|
handler_debug_stream.seek(0)
|
||||||
|
messages = handler_debug_stream.readlines()
|
||||||
|
|
||||||
|
for msg in messages:
|
||||||
|
self.assertNotIn("ERROR", msg)
|
||||||
|
|
||||||
|
if "INFO: Completed job " in msg:
|
||||||
|
job_id = msg.replace("INFO: Completed job ", "")
|
||||||
|
job_id = job_id[:job_id.index(" with output")]
|
||||||
|
loops = 15
|
||||||
|
loops += 1
|
||||||
|
|
||||||
|
self.assertIsNotNone(job_id)
|
||||||
|
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
|
||||||
|
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
|
||||||
|
|
||||||
|
runner.stop()
|
||||||
|
|
||||||
|
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
|
||||||
|
self.assertEqual(len(os.listdir(job_dir)), 5)
|
||||||
|
|
||||||
|
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
|
||||||
|
output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
|
||||||
|
self.assertTrue(os.path.exists(output_path))
|
||||||
|
|
||||||
|
with open(output_path, "r") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
self.assertEqual(data, "Initial Data\nA line from a test Pattern")
|
||||||
|
|
||||||
|
def testMeowRunnerLinkeExecution(self)->None:
|
||||||
|
monitor_to_handler_reader, monitor_to_handler_writer = Pipe()
|
||||||
|
|
||||||
|
pattern_one = FileEventPattern(
|
||||||
|
"pattern_one", "start/A.txt", "recipe_one", "infile",
|
||||||
|
parameters={
|
||||||
|
"extra":"A line from Pattern 1",
|
||||||
|
"outfile":"{VGRID}/middle/{FILENAME}"
|
||||||
|
})
|
||||||
|
pattern_two = FileEventPattern(
|
||||||
|
"pattern_two", "middle/A.txt", "recipe_one", "infile",
|
||||||
|
parameters={
|
||||||
|
"extra":"A line from Pattern 2",
|
||||||
|
"outfile":"{VGRID}/output/{FILENAME}"
|
||||||
|
})
|
||||||
|
recipe = JupyterNotebookRecipe(
|
||||||
|
"recipe_one", APPENDING_NOTEBOOK)
|
||||||
|
|
||||||
|
patterns = {
|
||||||
|
pattern_one.name: pattern_one,
|
||||||
|
pattern_two.name: pattern_two,
|
||||||
|
}
|
||||||
|
recipes = {
|
||||||
|
recipe.name: recipe,
|
||||||
|
}
|
||||||
|
rules = create_rules(patterns, recipes)
|
||||||
|
|
||||||
|
monitor_debug_stream = io.StringIO("")
|
||||||
|
handler_debug_stream = io.StringIO("")
|
||||||
|
|
||||||
|
runner = MeowRunner(
|
||||||
|
WatchdogMonitor(
|
||||||
|
TEST_MONITOR_BASE,
|
||||||
|
rules,
|
||||||
|
monitor_to_handler_writer,
|
||||||
|
print=monitor_debug_stream,
|
||||||
|
logging=3, settletime=1
|
||||||
|
),
|
||||||
|
PapermillHandler(
|
||||||
|
[monitor_to_handler_reader],
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT,
|
||||||
|
print=handler_debug_stream,
|
||||||
|
logging=3
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
runner.start()
|
||||||
|
|
||||||
|
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
|
||||||
|
make_dir(start_dir)
|
||||||
|
self.assertTrue(start_dir)
|
||||||
|
with open(os.path.join(start_dir, "A.txt"), "w") as f:
|
||||||
|
f.write("Initial Data")
|
||||||
|
|
||||||
|
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
|
||||||
|
|
||||||
|
loops = 0
|
||||||
|
job_ids = []
|
||||||
|
while len(job_ids) < 2 or loops < 30:
|
||||||
|
sleep(1)
|
||||||
|
handler_debug_stream.seek(0)
|
||||||
|
messages = handler_debug_stream.readlines()
|
||||||
|
|
||||||
|
for msg in messages:
|
||||||
|
self.assertNotIn("ERROR", msg)
|
||||||
|
|
||||||
|
if "INFO: Completed job " in msg:
|
||||||
|
job_id = msg.replace("INFO: Completed job ", "")
|
||||||
|
job_id = job_id[:job_id.index(" with output")]
|
||||||
|
if job_id not in job_ids:
|
||||||
|
job_ids.append(job_id)
|
||||||
|
loops += 1
|
||||||
|
|
||||||
|
print(job_ids)
|
||||||
|
|
||||||
|
self.assertEqual(len(job_ids), 2)
|
||||||
|
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 2)
|
||||||
|
self.assertIn(job_ids[0], os.listdir(TEST_JOB_OUTPUT))
|
||||||
|
self.assertIn(job_ids[1], os.listdir(TEST_JOB_OUTPUT))
|
||||||
|
|
||||||
|
runner.stop()
|
||||||
|
|
||||||
|
mid_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
|
||||||
|
self.assertEqual(len(os.listdir(mid_job_dir)), 5)
|
||||||
|
|
||||||
|
result = read_notebook(os.path.join(mid_job_dir, RESULT_FILE))
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
|
||||||
|
mid_output_path = os.path.join(TEST_MONITOR_BASE, "middle", "A.txt")
|
||||||
|
self.assertTrue(os.path.exists(mid_output_path))
|
||||||
|
|
||||||
|
with open(mid_output_path, "r") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
self.assertEqual(data, "Initial Data\nA line from Pattern 1")
|
||||||
|
|
||||||
|
final_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
|
||||||
|
self.assertEqual(len(os.listdir(final_job_dir)), 5)
|
||||||
|
|
||||||
|
result = read_notebook(os.path.join(final_job_dir, RESULT_FILE))
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
|
||||||
|
final_output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
|
||||||
|
self.assertTrue(os.path.exists(final_output_path))
|
||||||
|
|
||||||
|
with open(final_output_path, "r") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
self.assertEqual(data,
|
||||||
|
"Initial Data\nA line from Pattern 1\nA line from Pattern 2")
|
||||||
|
|
||||||
|
@ -1,28 +1,24 @@
|
|||||||
|
|
||||||
import shutil
|
|
||||||
import os
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from multiprocessing import Pipe
|
from multiprocessing import Pipe
|
||||||
|
|
||||||
from core.correctness.vars import FILE_EVENTS, FILE_CREATE_EVENT, PIPE_READ, \
|
from core.correctness.vars import FILE_EVENTS, FILE_CREATE_EVENT, \
|
||||||
PIPE_WRITE, BAREBONES_NOTEBOOK
|
BAREBONES_NOTEBOOK, TEST_MONITOR_BASE
|
||||||
from core.functionality import create_rules
|
from core.functionality import create_rules, rmtree, make_dir
|
||||||
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor
|
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
|
||||||
|
_DEFAULT_MASK
|
||||||
from recipes import JupyterNotebookRecipe
|
from recipes import JupyterNotebookRecipe
|
||||||
|
|
||||||
TEST_BASE = "test_base"
|
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
super().setUp()
|
super().setUp()
|
||||||
if not os.path.exists(TEST_BASE):
|
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
|
||||||
os.mkdir(TEST_BASE)
|
|
||||||
|
|
||||||
def tearDown(self) -> None:
|
def tearDown(self) -> None:
|
||||||
super().tearDown()
|
super().tearDown()
|
||||||
if os.path.exists(TEST_BASE):
|
rmtree(TEST_MONITOR_BASE)
|
||||||
shutil.rmtree(TEST_BASE)
|
|
||||||
|
|
||||||
def testFileEventPatternCreationMinimum(self)->None:
|
def testFileEventPatternCreationMinimum(self)->None:
|
||||||
FileEventPattern("name", "path", "recipe", "file")
|
FileEventPattern("name", "path", "recipe", "file")
|
||||||
@ -95,7 +91,7 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
|
|
||||||
def testFileEventPatternEventMask(self)->None:
|
def testFileEventPatternEventMask(self)->None:
|
||||||
fep = FileEventPattern("name", "path", "recipe", "file")
|
fep = FileEventPattern("name", "path", "recipe", "file")
|
||||||
self.assertEqual(fep.event_mask, FILE_EVENTS)
|
self.assertEqual(fep.event_mask, _DEFAULT_MASK)
|
||||||
|
|
||||||
with self.assertRaises(TypeError):
|
with self.assertRaises(TypeError):
|
||||||
fep = FileEventPattern("name", "path", "recipe", "file",
|
fep = FileEventPattern("name", "path", "recipe", "file",
|
||||||
@ -109,11 +105,9 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
fep = FileEventPattern("name", "path", "recipe", "file",
|
fep = FileEventPattern("name", "path", "recipe", "file",
|
||||||
event_mask=[FILE_CREATE_EVENT, "nope"])
|
event_mask=[FILE_CREATE_EVENT, "nope"])
|
||||||
|
|
||||||
self.assertEqual(fep.event_mask, FILE_EVENTS)
|
|
||||||
|
|
||||||
def testWatchdogMonitorMinimum(self)->None:
|
def testWatchdogMonitorMinimum(self)->None:
|
||||||
from_monitor = Pipe()
|
from_monitor = Pipe()
|
||||||
WatchdogMonitor(TEST_BASE, {}, from_monitor[PIPE_WRITE])
|
WatchdogMonitor(TEST_MONITOR_BASE, {}, from_monitor[1])
|
||||||
|
|
||||||
def testWatchdogMonitorEventIdentificaion(self)->None:
|
def testWatchdogMonitorEventIdentificaion(self)->None:
|
||||||
from_monitor_reader, from_monitor_writer = Pipe()
|
from_monitor_reader, from_monitor_writer = Pipe()
|
||||||
@ -131,11 +125,11 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
}
|
}
|
||||||
rules = create_rules(patterns, recipes)
|
rules = create_rules(patterns, recipes)
|
||||||
|
|
||||||
wm = WatchdogMonitor(TEST_BASE, rules, from_monitor_writer)
|
wm = WatchdogMonitor(TEST_MONITOR_BASE, rules, from_monitor_writer)
|
||||||
|
|
||||||
wm.start()
|
wm.start()
|
||||||
|
|
||||||
open(os.path.join(TEST_BASE, "A"), "w")
|
open(os.path.join(TEST_MONITOR_BASE, "A"), "w")
|
||||||
if from_monitor_reader.poll(3):
|
if from_monitor_reader.poll(3):
|
||||||
message = from_monitor_reader.recv()
|
message = from_monitor_reader.recv()
|
||||||
|
|
||||||
@ -143,9 +137,9 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
event, rule = message
|
event, rule = message
|
||||||
self.assertIsNotNone(event)
|
self.assertIsNotNone(event)
|
||||||
self.assertIsNotNone(rule)
|
self.assertIsNotNone(rule)
|
||||||
self.assertEqual(event.src_path, os.path.join(TEST_BASE, "A"))
|
self.assertEqual(event.src_path, os.path.join(TEST_MONITOR_BASE, "A"))
|
||||||
|
|
||||||
open(os.path.join(TEST_BASE, "B"), "w")
|
open(os.path.join(TEST_MONITOR_BASE, "B"), "w")
|
||||||
if from_monitor_reader.poll(3):
|
if from_monitor_reader.poll(3):
|
||||||
new_message = from_monitor_reader.recv()
|
new_message = from_monitor_reader.recv()
|
||||||
else:
|
else:
|
||||||
|
@ -1,19 +1,33 @@
|
|||||||
|
|
||||||
|
import io
|
||||||
import jsonschema
|
import jsonschema
|
||||||
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from multiprocessing import Pipe
|
from multiprocessing import Pipe
|
||||||
|
from time import sleep
|
||||||
|
from watchdog.events import FileCreatedEvent
|
||||||
|
|
||||||
|
from patterns.file_event_pattern import FileEventPattern
|
||||||
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
|
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
|
||||||
PapermillHandler
|
PapermillHandler, BASE_FILE, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
|
||||||
from core.correctness.vars import BAREBONES_NOTEBOOK
|
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
|
||||||
|
from core.correctness.vars import BAREBONES_NOTEBOOK, TEST_HANDLER_BASE, \
|
||||||
|
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK
|
||||||
|
from core.functionality import rmtree, make_dir, create_rules, read_notebook
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self) -> None:
|
||||||
return super().setUp()
|
super().setUp()
|
||||||
|
make_dir(TEST_MONITOR_BASE)
|
||||||
|
make_dir(TEST_HANDLER_BASE)
|
||||||
|
make_dir(TEST_JOB_OUTPUT)
|
||||||
|
|
||||||
def tearDown(self)->None:
|
def tearDown(self) -> None:
|
||||||
return super().tearDown()
|
super().tearDown()
|
||||||
|
rmtree(TEST_MONITOR_BASE)
|
||||||
|
rmtree(TEST_HANDLER_BASE)
|
||||||
|
rmtree(TEST_JOB_OUTPUT)
|
||||||
|
|
||||||
def testJupyterNotebookRecipeCreationMinimum(self)->None:
|
def testJupyterNotebookRecipeCreationMinimum(self)->None:
|
||||||
JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
|
JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
|
||||||
@ -80,12 +94,20 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
def testPapermillHanderMinimum(self)->None:
|
def testPapermillHanderMinimum(self)->None:
|
||||||
monitor_to_handler_reader, _ = Pipe()
|
monitor_to_handler_reader, _ = Pipe()
|
||||||
|
|
||||||
PapermillHandler([monitor_to_handler_reader])
|
PapermillHandler(
|
||||||
|
[monitor_to_handler_reader],
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT
|
||||||
|
)
|
||||||
|
|
||||||
def testPapermillHanderStartStop(self)->None:
|
def testPapermillHanderStartStop(self)->None:
|
||||||
monitor_to_handler_reader, _ = Pipe()
|
monitor_to_handler_reader, _ = Pipe()
|
||||||
|
|
||||||
ph = PapermillHandler([monitor_to_handler_reader])
|
ph = PapermillHandler(
|
||||||
|
[monitor_to_handler_reader],
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT
|
||||||
|
)
|
||||||
|
|
||||||
ph.start()
|
ph.start()
|
||||||
ph.stop()
|
ph.stop()
|
||||||
@ -93,7 +115,11 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
def testPapermillHanderRepeatedStarts(self)->None:
|
def testPapermillHanderRepeatedStarts(self)->None:
|
||||||
monitor_to_handler_reader, _ = Pipe()
|
monitor_to_handler_reader, _ = Pipe()
|
||||||
|
|
||||||
ph = PapermillHandler([monitor_to_handler_reader])
|
ph = PapermillHandler(
|
||||||
|
[monitor_to_handler_reader],
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT
|
||||||
|
)
|
||||||
|
|
||||||
ph.start()
|
ph.start()
|
||||||
with self.assertRaises(RuntimeWarning):
|
with self.assertRaises(RuntimeWarning):
|
||||||
@ -103,9 +129,87 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
def testPapermillHanderStopBeforeStart(self)->None:
|
def testPapermillHanderStopBeforeStart(self)->None:
|
||||||
monitor_to_handler_reader, _ = Pipe()
|
monitor_to_handler_reader, _ = Pipe()
|
||||||
|
|
||||||
ph = PapermillHandler([monitor_to_handler_reader])
|
ph = PapermillHandler(
|
||||||
|
[monitor_to_handler_reader],
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT
|
||||||
|
)
|
||||||
|
|
||||||
with self.assertRaises(RuntimeWarning):
|
with self.assertRaises(RuntimeWarning):
|
||||||
ph.stop()
|
ph.stop()
|
||||||
|
|
||||||
|
def testPapermillHandlerHandling(self)->None:
|
||||||
|
monitor_to_handler_reader, to_handler = Pipe()
|
||||||
|
|
||||||
|
debug_stream = io.StringIO("")
|
||||||
|
|
||||||
|
ph = PapermillHandler(
|
||||||
|
[monitor_to_handler_reader],
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT,
|
||||||
|
print=debug_stream,
|
||||||
|
logging=3
|
||||||
|
)
|
||||||
|
|
||||||
|
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
|
||||||
|
f.write("Data")
|
||||||
|
event = FileCreatedEvent(os.path.join(TEST_MONITOR_BASE, "A"))
|
||||||
|
event.monitor_base = TEST_MONITOR_BASE
|
||||||
|
|
||||||
|
pattern_one = FileEventPattern(
|
||||||
|
"pattern_one", "A", "recipe_one", "file_one")
|
||||||
|
recipe = JupyterNotebookRecipe(
|
||||||
|
"recipe_one", COMPLETE_NOTEBOOK)
|
||||||
|
|
||||||
|
patterns = {
|
||||||
|
pattern_one.name: pattern_one,
|
||||||
|
}
|
||||||
|
recipes = {
|
||||||
|
recipe.name: recipe,
|
||||||
|
}
|
||||||
|
|
||||||
|
rules = create_rules(patterns, recipes)
|
||||||
|
self.assertEqual(len(rules), 1)
|
||||||
|
_, rule = rules.popitem()
|
||||||
|
self.assertIsInstance(rule, FileEventJupyterNotebookRule)
|
||||||
|
|
||||||
|
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
|
||||||
|
|
||||||
|
ph.start()
|
||||||
|
to_handler.send((event, rule))
|
||||||
|
|
||||||
|
loops = 0
|
||||||
|
job_id = None
|
||||||
|
while loops < 15:
|
||||||
|
sleep(1)
|
||||||
|
debug_stream.seek(0)
|
||||||
|
messages = debug_stream.readlines()
|
||||||
|
|
||||||
|
for msg in messages:
|
||||||
|
self.assertNotIn("ERROR", msg)
|
||||||
|
|
||||||
|
if "INFO: Completed job " in msg:
|
||||||
|
job_id = msg.replace("INFO: Completed job ", "")
|
||||||
|
job_id = job_id[:job_id.index(" with output")]
|
||||||
|
loops = 15
|
||||||
|
loops += 1
|
||||||
|
|
||||||
|
self.assertIsNotNone(job_id)
|
||||||
|
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
|
||||||
|
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
|
||||||
|
|
||||||
|
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
|
||||||
|
self.assertEqual(len(os.listdir(job_dir)), 5)
|
||||||
|
|
||||||
|
self.assertIn(META_FILE, os.listdir(job_dir))
|
||||||
|
self.assertIn(BASE_FILE, os.listdir(job_dir))
|
||||||
|
self.assertIn(PARAMS_FILE, os.listdir(job_dir))
|
||||||
|
self.assertIn(JOB_FILE, os.listdir(job_dir))
|
||||||
|
self.assertIn(RESULT_FILE, os.listdir(job_dir))
|
||||||
|
|
||||||
|
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
|
||||||
|
|
||||||
|
self.assertEqual("124875.0\n",
|
||||||
|
result["cells"][4]["outputs"][0]["text"][0])
|
||||||
|
|
||||||
|
ph.stop()
|
||||||
|
@ -1,19 +1,23 @@
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
import os
|
||||||
|
|
||||||
from typing import Any, Union
|
from typing import Any, Union
|
||||||
|
|
||||||
from core.correctness.validation import check_type, check_implementation, \
|
from core.correctness.validation import check_type, check_implementation, \
|
||||||
valid_string, valid_dict, valid_list
|
valid_string, valid_dict, valid_list, valid_existing_file_path, \
|
||||||
from core.correctness.vars import VALID_NAME_CHARS
|
valid_existing_dir_path, valid_non_existing_path
|
||||||
|
from core.correctness.vars import VALID_NAME_CHARS, TEST_MONITOR_BASE, SHA256
|
||||||
|
from core.functionality import rmtree, make_dir
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self) -> None:
|
||||||
return super().setUp()
|
super().setUp()
|
||||||
|
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
|
||||||
|
|
||||||
def tearDown(self)->None:
|
def tearDown(self) -> None:
|
||||||
return super().tearDown()
|
super().tearDown()
|
||||||
|
rmtree(TEST_MONITOR_BASE)
|
||||||
|
|
||||||
def testCheckTypeValid(self)->None:
|
def testCheckTypeValid(self)->None:
|
||||||
check_type(1, int)
|
check_type(1, int)
|
||||||
@ -158,3 +162,45 @@ class CorrectnessTests(unittest.TestCase):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
check_implementation(Child.func, Parent)
|
check_implementation(Child.func, Parent)
|
||||||
|
|
||||||
|
def testValidExistingFilePath(self)->None:
|
||||||
|
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
|
||||||
|
with open(file_path, 'w') as hashed_file:
|
||||||
|
hashed_file.write("Some data\n")
|
||||||
|
|
||||||
|
valid_existing_file_path(file_path)
|
||||||
|
|
||||||
|
with self.assertRaises(FileNotFoundError):
|
||||||
|
valid_existing_file_path("not_existing_"+file_path, SHA256)
|
||||||
|
|
||||||
|
dir_path = os.path.join(TEST_MONITOR_BASE, "dir")
|
||||||
|
make_dir(dir_path)
|
||||||
|
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
valid_existing_file_path(dir_path, SHA256)
|
||||||
|
|
||||||
|
def testValidExistingDirPath(self)->None:
|
||||||
|
valid_existing_dir_path(TEST_MONITOR_BASE)
|
||||||
|
|
||||||
|
dir_path = os.path.join(TEST_MONITOR_BASE, "dir")
|
||||||
|
|
||||||
|
with self.assertRaises(FileNotFoundError):
|
||||||
|
valid_existing_dir_path("not_existing_"+dir_path, SHA256)
|
||||||
|
|
||||||
|
file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
|
||||||
|
with open(file_path, 'w') as hashed_file:
|
||||||
|
hashed_file.write("Some data\n")
|
||||||
|
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
valid_existing_dir_path(file_path, SHA256)
|
||||||
|
|
||||||
|
def testValidNonExistingPath(self)->None:
|
||||||
|
valid_non_existing_path("does_not_exist")
|
||||||
|
|
||||||
|
make_dir("first")
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
valid_non_existing_path("first")
|
||||||
|
|
||||||
|
make_dir("first/second")
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
valid_non_existing_path("first/second")
|
||||||
|
Reference in New Issue
Block a user