NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated job creation to be written to files, which are then read and executed

Browse Source
This commit is contained in:
PatchOfScotland
2023-01-31 18:03:11 +01:00
parent bb7c4af1e5
commit a2abf91e7e
4 changed files with 109 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
tests/test_conductors.py
View File

@ -6,7 +6,8 @@ from core.correctness.vars import PYTHON_TYPE, SHA256, WATCHDOG_TYPE, \
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
from core.functionality import get_file_hash, create_event, create_job
from core.functionality import get_file_hash, create_event, create_job, \
make_dir, write_yaml, write_notebook
from core.meow import create_rule
from conductors import LocalPythonConductor
from patterns import FileEventPattern
@ -62,6 +63,12 @@ class MeowTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
params_dict = {
"extra":"extra",
"infile":file_path,
"outfile":result_path
}
job_dict = create_job(
PYTHON_TYPE,
create_event(
@ -74,11 +81,7 @@ class MeowTests(unittest.TestCase):
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
@ -86,6 +89,15 @@ class MeowTests(unittest.TestCase):
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
@ -127,6 +139,12 @@ class MeowTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
params_dict = {
"extra":"extra",
"infile":file_path,
"outfile":result_path
}
bad_job_dict = create_job(
PYTHON_TYPE,
create_event(
@ -139,16 +157,21 @@ class MeowTests(unittest.TestCase):
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, bad_job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
write_notebook(APPENDING_NOTEBOOK, base_file)
with self.assertRaises(KeyError):
lpc.execute(bad_job_dict)
@ -165,11 +188,7 @@ class MeowTests(unittest.TestCase):
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
@ -177,6 +196,15 @@ class MeowTests(unittest.TestCase):
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, good_job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(good_job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, good_job_dict[JOB_ID])

25
tests/test_recipes.py
View File

@ -11,7 +11,8 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, WATCHDOG_RULE, \
PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, \
RESULT_FILE
from core.correctness.validation import valid_job
from core.functionality import get_file_hash, create_job, create_event
from core.functionality import get_file_hash, create_job, create_event, \
make_dir, write_yaml, write_notebook
from core.meow import create_rules, create_rule
from patterns.file_event_pattern import FileEventPattern, SWEEP_START, \
SWEEP_STOP, SWEEP_JUMP
@ -332,6 +333,12 @@ class CorrectnessTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
params_dict = {
"extra":"extra",
"infile":file_path,
"outfile":result_path
}
job_dict = create_job(
PYTHON_TYPE,
create_event(
@ -344,11 +351,7 @@ class CorrectnessTests(unittest.TestCase):
}
),
{
JOB_PARAMETERS:{
"extra":"extra",
"infile":file_path,
"outfile":result_path
},
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
@ -356,6 +359,16 @@ class CorrectnessTests(unittest.TestCase):
}
)
job_dir = os.path.join(
job_dict[PYTHON_EXECUTION_BASE], job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
write_notebook(APPENDING_NOTEBOOK, base_file)
job_func(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])

3
tests/test_runner.py
View File

@ -10,8 +10,7 @@ from core.correctness.vars import RESULT_FILE
from core.functionality import make_dir, read_notebook
from core.meow import BaseMonitor, BaseHandler, BaseConductor
from core.runner import MeowRunner
from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern, \
SWEEP_JUMP, SWEEP_START, SWEEP_STOP
from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
from recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe
from shared import setup, teardown, TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \