NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated code so should work on windows, with exception of waiting on multiple connections

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-08 14:46:03 +01:00
parent d787e37adc
commit a1451881ae
17 changed files with 280 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tests/shared.py
View File

@ -3,6 +3,8 @@ This file contains shared functions and variables used within multiple tests.
Author(s): David Marchant
"""
import os
from core.functionality import make_dir, rmtree
@ -30,8 +32,8 @@ COMPLETE_PYTHON_SCRIPT = [
"import os",
"# Setup parameters",
"num = 1000",
"infile = 'somehere/particular'",
"outfile = 'nowhere/particular'",
"infile = 'somehere"+ os.path.sep +"particular'",
"outfile = 'nowhere"+ os.path.sep +"particular'",
"",
"with open(infile, 'r') as file:",
" s = float(file.read())",
@ -131,9 +133,9 @@ APPENDING_NOTEBOOK = {
"# The line to append\n",
"extra = 'This line comes from a default pattern'\n",
"# Data input file location\n",
"infile = 'start/alpha.txt'\n",
"infile = 'start"+ os.path.sep +"alpha.txt'\n",
"# Output file location\n",
"outfile = 'first/alpha.txt'"
"outfile = 'first"+ os.path.sep +"alpha.txt'"
]
},
{
@ -216,9 +218,9 @@ ADDING_NOTEBOOK = {
"# Amount to add to data\n",
"extra = 10\n",
"# Data input file location\n",
"infile = 'example_data/data_0.npy'\n",
"infile = 'example_data"+ os.path.sep +"data_0.npy'\n",
"# Output file location\n",
"outfile = 'standard_output/data_0.npy'"
"outfile = 'standard_output"+ os.path.sep +"data_0.npy'"
]
},
{

6
tests/test_conductors.py
View File

@ -2,6 +2,8 @@
import os
import unittest
from typing import Dict
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
@ -108,7 +110,7 @@ class MeowTests(unittest.TestCase):
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
@ -196,7 +198,7 @@ class MeowTests(unittest.TestCase):
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)

77
tests/test_functionality.py
View File

@ -383,28 +383,38 @@ class CorrectnessTests(unittest.TestCase):
}
replaced = replace_keywords(
test_dict, "job_id", "base/src/dir/file.ext", "base/monitor/dir")
test_dict,
"job_id",
os.path.join("base", "src", "dir", "file.ext"),
os.path.join("base", "monitor", "dir")
)
self.assertIsInstance(replaced, dict)
self.assertEqual(len(test_dict.keys()), len(replaced.keys()))
for k in test_dict.keys():
self.assertIn(k, replaced)
self.assertEqual(replaced["A"], "--base/src/dir/file.ext--")
self.assertEqual(replaced["B"], "--../../src/dir/file.ext--")
self.assertEqual(replaced["C"], "--base/src/dir--")
self.assertEqual(replaced["D"], "--../../src/dir--")
self.assertEqual(replaced["A"],
os.path.join("--base", "src", "dir", "file.ext--"))
self.assertEqual(replaced["B"],
os.path.join("--..", "..", "src", "dir", "file.ext--"))
self.assertEqual(replaced["C"],
os.path.join("--base", "src", "dir--"))
self.assertEqual(replaced["D"],
os.path.join("--..", "..", "src", "dir--"))
self.assertEqual(replaced["E"], "--file.ext--")
self.assertEqual(replaced["F"], "--file--")
self.assertEqual(replaced["G"], "--base/monitor/dir--")
self.assertEqual(replaced["G"],
os.path.join("--base", "monitor", "dir--"))
self.assertEqual(replaced["H"], "--.ext--")
self.assertEqual(replaced["I"], "--job_id--")
self.assertEqual(replaced["J"],
"--base/src/dir/file.ext-base/src/dir/file.ext--")
self.assertEqual(replaced["K"], "base/src/dir/file.ext")
os.path.join("--base", "src", "dir", "file.ext-base", "src", "dir", "file.ext--"))
self.assertEqual(replaced["K"],
os.path.join("base", "src", "dir", "file.ext"))
self.assertEqual(replaced["L"],
"--base/src/dir/file.ext-../../src/dir/file.ext-base/src/dir-"
"../../src/dir-file.ext-file-base/monitor/dir-.ext-job_id--")
os.path.join("--base", "src", "dir", "file.ext-..", "..", "src", "dir", "file.ext-base", "src", "dir-"
"..", "..", "src", "dir-file.ext-file-base", "monitor", "dir-.ext-job_id--"))
self.assertEqual(replaced["M"], "A")
self.assertEqual(replaced["N"], 1)
@ -498,30 +508,31 @@ data"""
'"metadata": {}, "outputs": [], "source": ["# Default parameters '
'values\\n", "# The line to append\\n", "extra = \'This line '
'comes from a default pattern\'\\n", "# Data input file '
'location\\n", "infile = \'start/alpha.txt\'\\n", "# Output file '
'location\\n", "outfile = \'first/alpha.txt\'"]}, {"cell_type": '
'location\\n", "infile = \'start'+ os.path.sep +'alpha.txt\'\\n", '
'"# Output file location\\n", "outfile = \'first'+ os.path.sep
+'alpha.txt\'"]}, {"cell_type": "code", "execution_count": null, '
'"metadata": {}, "outputs": [], "source": ["# load in dataset. '
'This should be a text file\\n", "with open(infile) as '
'input_file:\\n", " data = input_file.read()"]}, {"cell_type": '
'"code", "execution_count": null, "metadata": {}, "outputs": [], '
'"source": ["# load in dataset. This should be a text file\\n", '
'"with open(infile) as input_file:\\n", " data = '
'input_file.read()"]}, {"cell_type": "code", "execution_count": '
'null, "metadata": {}, "outputs": [], "source": ["# Append the '
'line\\n", "appended = data + \'\\\\n\' + extra"]}, {"cell_type": '
'"code", "execution_count": null, "metadata": {}, "outputs": [], '
'"source": ["import os\\n", "\\n", "# Create output directory if '
'it doesn\'t exist\\n", "output_dir_path = '
'os.path.dirname(outfile)\\n", "\\n", "if output_dir_path:\\n", '
'" os.makedirs(output_dir_path, exist_ok=True)\\n", "\\n", "# '
'Save added array as new dataset\\n", "with open(outfile, \'w\') '
'as output_file:\\n", " output_file.write(appended)"]}], '
'"metadata": {"kernelspec": {"display_name": "Python 3", '
'"language": "python", "name": "python3"}, "language_info": '
'{"codemirror_mode": {"name": "ipython", "version": 3}, '
'"file_extension": ".py", "mimetype": "text/x-python", "name": '
'"python", "nbconvert_exporter": "python", "pygments_lexer": '
'"ipython3", "version": "3.10.6 (main, Nov 14 2022, 16:10:14) '
'[GCC 11.3.0]"}, "vscode": {"interpreter": {"hash": '
'"916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1'
'"}}}, "nbformat": 4, "nbformat_minor": 4}'
'"source": ["# Append the line\\n", "appended = data + \'\\\\n\' '
'+ extra"]}, {"cell_type": "code", "execution_count": null, '
'"metadata": {}, "outputs": [], "source": ["import os\\n", "\\n", '
'"# Create output directory if it doesn\'t exist\\n", '
'"output_dir_path = os.path.dirname(outfile)\\n", "\\n", '
'"if output_dir_path:\\n", " os.makedirs(output_dir_path, '
'exist_ok=True)\\n", "\\n", "# Save added array as new '
'dataset\\n", "with open(outfile, \'w\') as output_file:\\n", " '
'output_file.write(appended)"]}], "metadata": {"kernelspec": '
'{"display_name": "Python 3", "language": "python", "name": '
'"python3"}, "language_info": {"codemirror_mode": {"name": '
'"ipython", "version": 3}, "file_extension": ".py", "mimetype": '
'"text/x-python", "name": "python", "nbconvert_exporter": '
'"python", "pygments_lexer": "ipython3", "version": "3.10.6 '
'(main, Nov 14 2022, 16:10:14) [GCC 11.3.0]"}, "vscode": '
'{"interpreter": {"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c'
'65e1c03b4aa3499c5328201f1"}}}, "nbformat": 4, "nbformat_minor": '
'4}'
]
self.assertEqual(data, expected_bytes)

16
tests/test_meow.py
View File

@ -1,7 +1,7 @@
import unittest
from typing import Any, Union, Tuple
from typing import Any, Union, Tuple, Dict
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
@ -67,7 +67,7 @@ class MeowTests(unittest.TestCase):
def _is_valid_output(self, outputs:Any)->None:
pass
def _is_valid_sweep(self,
sweep:dict[str,Union[int,float,complex]])->None:
sweep:Dict[str,Union[int,float,complex]])->None:
pass
FullPattern("name", "", "", "", "")
@ -188,7 +188,7 @@ class MeowTests(unittest.TestCase):
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, dict)
self.assertIsInstance(rules, Dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
@ -229,9 +229,9 @@ class MeowTests(unittest.TestCase):
pass
def stop(self):
pass
def _is_valid_patterns(self, patterns:dict[str,BasePattern])->None:
def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
pass
def _is_valid_recipes(self, recipes:dict[str,BaseRecipe])->None:
def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
pass
def add_pattern(self, pattern:BasePattern)->None:
pass
@ -274,7 +274,7 @@ class MeowTests(unittest.TestCase):
pass
def _is_valid_inputs(self, inputs:Any)->None:
pass
def valid_handle_criteria(self, event:dict[str,Any]
def valid_handle_criteria(self, event:Dict[str,Any]
)->Tuple[bool,str]:
pass
@ -292,10 +292,10 @@ class MeowTests(unittest.TestCase):
TestConductor()
class FullTestConductor(BaseConductor):
def execute(self, job:dict[str,Any])->None:
def execute(self, job:Dict[str,Any])->None:
pass
def valid_execute_criteria(self, job:dict[str,Any]
def valid_execute_criteria(self, job:Dict[str,Any]
)->Tuple[bool,str]:
pass

60
tests/test_patterns.py
View File

@ -245,7 +245,10 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor identifies expected events in sub directories
def testMonitoring(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
@ -304,7 +307,10 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor identifies fake events for retroactive patterns
def testMonitoringRetroActive(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
@ -368,10 +374,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor get_patterns function
def testMonitorGetPatterns(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -395,10 +407,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor add_pattern function
def testMonitorAddPattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -440,10 +458,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor update_patterns function
def testMonitorUpdatePattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -502,10 +526,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor remove_patterns function
def testMonitorRemovePattern(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
wm = WatchdogMonitor(
@ -707,10 +737,16 @@ class CorrectnessTests(unittest.TestCase):
# Test WatchdogMonitor get_rules function
def testMonitorGetRules(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={})
pattern_two = FileEventPattern(
"pattern_two", "start/B.txt", "recipe_two", "infile",
"pattern_two",
os.path.join("start", "B.txt"),
"recipe_two",
"infile",
parameters={})
recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)

5
tests/test_recipes.py
View File

@ -4,6 +4,7 @@ import os
import unittest
from multiprocessing import Pipe
from typing import Dict
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
@ -391,7 +392,7 @@ class JupyterNotebookTests(unittest.TestCase):
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
@ -759,7 +760,7 @@ class PythonTests(unittest.TestCase):
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
self.assertNotIn(JOB_ERROR, status)

30
tests/test_runner.py
View File

@ -121,10 +121,13 @@ class MeowTests(unittest.TestCase):
# Test single meow papermill job execution
def testMeowRunnerPapermillExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
@ -205,16 +208,19 @@ class MeowTests(unittest.TestCase):
# Test meow papermill job chaining within runner
def testMeowRunnerLinkedPapermillExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one",
os.path.join("start", "A.txt"),
"recipe_one",
"infile",
parameters={
"extra":"A line from Pattern 1",
"outfile":"{VGRID}/middle/{FILENAME}"
"outfile":os.path.join("{VGRID}", "middle", "{FILENAME}")
})
pattern_two = FileEventPattern(
"pattern_two", "middle/A.txt", "recipe_one", "infile",
"pattern_two", os.path.join("middle", "A.txt"), "recipe_one", "infile",
parameters={
"extra":"A line from Pattern 2",
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK)
@ -314,10 +320,10 @@ class MeowTests(unittest.TestCase):
# Test single meow python job execution
def testMeowRunnerPythonExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one", os.path.join("start", "A.txt"), "recipe_one", "infile",
parameters={
"num":10000,
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT
@ -403,16 +409,16 @@ class MeowTests(unittest.TestCase):
# Test meow python job chaining within runner
def testMeowRunnerLinkedPythonExecution(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile",
"pattern_one", os.path.join("start", "A.txt"), "recipe_one", "infile",
parameters={
"num":250,
"outfile":"{VGRID}/middle/{FILENAME}"
"outfile":os.path.join("{VGRID}", "middle", "{FILENAME}")
})
pattern_two = FileEventPattern(
"pattern_two", "middle/A.txt", "recipe_one", "infile",
"pattern_two", os.path.join("middle", "A.txt"), "recipe_one", "infile",
parameters={
"num":40,
"outfile":"{VGRID}/output/{FILENAME}"
"outfile":os.path.join("{VGRID}", "output", "{FILENAME}")
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT

5
tests/test_validation.py
View File

@ -239,9 +239,10 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(ValueError):
valid_non_existing_path("first")
make_dir("first/second")
test_path = os.path.join("first", "second")
make_dir(test_path)
with self.assertRaises(ValueError):
valid_non_existing_path("first/second")
valid_non_existing_path(test_path)
# TODO modify so tests for actual rule values
# Test valid_event can check given event dictionary