NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

differentiated papermill and python jobs more clearly

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-03 14:47:16 +01:00
parent 72d6b263b7
commit 47f9fe73fa
17 changed files with 853 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
conductors/local_python_conductor.py
View File

@ -14,7 +14,7 @@ from typing import Any, Tuple
from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, PYTHON_OUTPUT_DIR, \
JOB_TYPE
JOB_TYPE, JOB_TYPE_PAPERMILL
from core.correctness.validation import valid_job
from core.functionality import read_yaml, write_yaml
from core.meow import BaseConductor
@ -30,7 +30,7 @@ class LocalPythonConductor(BaseConductor):
process it or not. This conductor will accept any Python job type"""
try:
valid_job(job)
if job[JOB_TYPE] == JOB_TYPE_PYTHON:
if job[JOB_TYPE] in [JOB_TYPE_PYTHON, JOB_TYPE_PAPERMILL]:
return True, ""
except Exception as e:
pass

14
core/correctness/validation.py
View File

@ -72,6 +72,12 @@ def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
% (get_args(expected_type), type(variable))
)
def check_callable(call:Any)->None:
"""Checks if a given variable is a callable function. Raises TypeError if
not."""
if not callable(call):
raise TypeError(f"Given object '{call}' is not a callable function")
def check_implementation(child_func, parent_class):
"""Checks if the given function has been overridden from the one inherited
from the parent class. Raises a NotImplementedError if this is the case."""
@ -94,6 +100,14 @@ def check_implementation(child_func, parent_class):
msg = get_not_imp_msg(parent_class, parent_func)
raise NotImplementedError(msg)
def check_script(script:Any):
"""Checks if a given variable is a valid script. Raises TypeError if
not."""
# TODO investigate more robust check here
check_type(script, list)
for line in script:
check_type(line, str)
def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
"""Checks that all characters in a given string are present in a provided
list of characters. Will raise an ValueError if unexpected character is

26
core/correctness/vars.py
View File

@ -80,10 +80,24 @@ DIR_EVENTS = [
# meow jobs
JOB_TYPE = "job_type"
JOB_TYPE_PYTHON = "python"
JOB_TYPE_PAPERMILL = "papermill"
PYTHON_FUNC = "func"
PYTHON_EXECUTION_BASE = "exection_base"
PYTHON_OUTPUT_DIR = "output_dir"
JOB_TYPES = {
JOB_TYPE_PAPERMILL: [
"base.ipynb",
"job.ipynb",
"result.ipynb",
],
JOB_TYPE_PYTHON: [
"base.py",
"job.py",
"result.py",
]
}
# job definitions
JOB_ID = "id"
JOB_EVENT = "event"
@ -108,10 +122,7 @@ STATUS_DONE = "done"
# job definition files
META_FILE = "job.yml"
BASE_FILE = "base.ipynb"
PARAMS_FILE = "params.yml"
JOB_FILE = "job.ipynb"
RESULT_FILE = "result.ipynb"
# Parameter sweep keys
SWEEP_START = "start"
@ -132,3 +143,12 @@ def get_not_imp_msg(parent_class, class_function):
return f"Children of the '{parent_class.__name__}' class must implement " \
f"the '{class_function.__name__}({signature(class_function)})' " \
"function"
def get_base_file(job_type:str):
return JOB_TYPES[job_type][0]
def get_job_file(job_type:str):
return JOB_TYPES[job_type][1]
def get_result_file(job_type:str):
return JOB_TYPES[job_type][2]

53
core/functionality.py
View File

@ -1,4 +1,4 @@
# TODO comments
import copy
import hashlib
import json
@ -15,7 +15,7 @@ from typing import Any
from random import SystemRandom
from core.correctness.validation import check_type, valid_existing_file_path, \
valid_path
valid_path, check_script
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
VALID_CHANNELS, HASH_BUFFER_SIZE, SHA256, DEBUG_WARNING, DEBUG_INFO, \
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
@ -128,6 +128,18 @@ def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):
os.makedirs(path, exist_ok=can_exist)
def read_file(filepath:str):
with open(filepath, 'r') as file:
return file.read()
def read_file_lines(filepath:str):
with open(filepath, 'r') as file:
return file.readlines()
def write_file(source:str, filename:str):
with open(filename, 'w') as file:
file.write(source)
def read_yaml(filepath:str):
"""
Reads a file path as a yaml object.
@ -171,7 +183,7 @@ def write_notebook(source:dict[str,Any], filename:str):
json.dump(source, job_file)
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
def parameterize_jupyter_notebook( jupyter_notebook:dict[str,Any],
def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any],
parameters:dict[str,Any], expand_env_values:bool=False)->dict[str,Any]:
nbformat.validate(jupyter_notebook)
check_type(parameters, dict)
@ -244,6 +256,38 @@ def parameterize_jupyter_notebook( jupyter_notebook:dict[str,Any],
return output_notebook
def parameterize_python_script(script:list[str], parameters:dict[str,Any],
expand_env_values:bool=False)->dict[str,Any]:
check_script(script)
check_type(parameters, dict)
output_script = copy.deepcopy(script)
for i, line in enumerate(output_script):
if "=" in line:
d_line = list(map(lambda x: x.replace(" ", ""),
line.split("=")))
# Matching parameter name
if len(d_line) == 2 and d_line[0] in parameters:
value = parameters[d_line[0]]
# Whether to expand value from os env
if (
expand_env_values
and isinstance(value, str)
and value.startswith("ENV_")
):
env_var = value.replace("ENV_", "")
value = os.getenv(
env_var,
"MISSING ENVIRONMENT VARIABLE: {}".format(env_var)
)
output_script[i] = f"{d_line[0]} = {repr(value)}"
# Validate that the parameterized notebook is still valid
check_script(output_script)
return output_script
def print_debug(print_target, debug_level, msg, level)->None:
if print_target is None:
return
@ -338,3 +382,6 @@ def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
}
return {**extras, **job_dict}
def lines_to_string(lines:list[str])->str:
return "\n".join(lines)

14
core/meow.py
View File

@ -98,7 +98,6 @@ class BasePattern:
check_implementation(type(self)._is_valid_recipe, BasePattern)
check_implementation(type(self)._is_valid_parameters, BasePattern)
check_implementation(type(self)._is_valid_output, BasePattern)
check_implementation(type(self)._is_valid_sweep, BasePattern)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
@ -140,8 +139,9 @@ class BasePattern:
pass
def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None:
"""Validation check for 'sweep' variable from main constructor. Must
be implemented by any child class."""
"""Validation check for 'sweep' variable from main constructor. This
function is implemented to check for the types given in the signature,
and must be overridden if these differ."""
check_type(sweep, dict)
if not sweep:
return
@ -208,13 +208,19 @@ class BaseRule:
the input parameters."""
check_implementation(type(self)._is_valid_pattern, BaseRule)
check_implementation(type(self)._is_valid_recipe, BaseRule)
self.__check_types_set()
self._is_valid_name(name)
self.name = name
self._is_valid_pattern(pattern)
self.pattern = pattern
self._is_valid_recipe(recipe)
self.recipe = recipe
self.__check_types_set()
check_type(pattern, BasePattern)
check_type(recipe, BaseRecipe)
if pattern.recipe != recipe.name:
raise ValueError(f"Cannot create Rule {name}. Pattern "
f"{pattern.name} does not identify Recipe {recipe.name}. It "
f"uses {pattern.recipe}")
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only

4
patterns/file_event_pattern.py
View File

@ -23,8 +23,8 @@ from core.correctness.validation import check_type, valid_string, \
setup_debugging
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, FILE_RETROACTIVE_EVENT, WATCHDOG_HASH, SHA256
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
FILE_RETROACTIVE_EVENT, SHA256
from core.functionality import print_debug, create_watchdog_event, \
get_file_hash, create_fake_watchdog_event
from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \

38
recipes/jupyter_notebook_recipe.py
View File

@ -6,7 +6,6 @@ notebooks, along with an appropriate handler for said events.
Author(s): David Marchant
"""
import os
import itertools
import nbformat
import sys
@ -17,12 +16,12 @@ from core.correctness.validation import check_type, valid_string, \
valid_event
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \
EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \
EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \
PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
EVENT_RULE
EVENT_RULE, get_base_file, get_job_file, get_result_file
from core.functionality import print_debug, create_job, replace_keywords, \
make_dir, write_yaml, write_notebook
make_dir, write_yaml, write_notebook, read_notebook
from core.meow import BaseRecipe, BaseHandler
@ -140,19 +139,19 @@ class PapermillHandler(BaseHandler):
"""Function to set up new job dict and send it to the runner to be
executed."""
meow_job = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
event,
extras={
JOB_PARAMETERS:yaml_dict,
JOB_HASH: event[WATCHDOG_HASH],
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:self.output_dir,
PYTHON_EXECUTION_BASE:self.handler_base
}
)
print_debug(self._print_target, self.debug_level,
f"Creating job from event at {event[EVENT_PATH]} of type "
f"{JOB_TYPE_PYTHON}.", DEBUG_INFO)
f"{JOB_TYPE_PAPERMILL}.", DEBUG_INFO)
# replace MEOW keyworks within variables dict
yaml_dict = replace_keywords(
@ -172,7 +171,7 @@ class PapermillHandler(BaseHandler):
write_yaml(meow_job, meta_file)
# write an executable notebook to the job directory
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(event[EVENT_RULE].recipe.recipe, base_file)
# write a parameter file to the job directory
@ -188,25 +187,26 @@ class PapermillHandler(BaseHandler):
self.to_runner.send(job_dir)
# Papermill job execution code, to be run within the conductor
def job_func(job):
def papermill_job_func(job):
# Requires own imports as will be run in its own execution environment
import os
import papermill
from datetime import datetime
from core.functionality import write_yaml, read_yaml, write_notebook, \
get_file_hash, parameterize_jupyter_notebook
from core.correctness.vars import JOB_EVENT, EVENT_RULE, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE, \
from core.correctness.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE
event = job[JOB_EVENT]
JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE, get_job_file, \
get_result_file
# Identify job files
job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID])
meta_file = os.path.join(job_dir, META_FILE)
job_file = os.path.join(job_dir, JOB_FILE)
result_file = os.path.join(job_dir, RESULT_FILE)
# TODO fix these paths so they are dynamic
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL))
result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL))
param_file = os.path.join(job_dir, PARAMS_FILE)
yaml_dict = read_yaml(param_file)
@ -233,8 +233,10 @@ def job_func(job):
# Create a parameterised version of the executable notebook
try:
base_notebook = read_notebook(base_file)
# TODO read notebook from already written file rather than event
job_notebook = parameterize_jupyter_notebook(
event[EVENT_RULE].recipe.recipe, yaml_dict
base_notebook, yaml_dict
)
write_notebook(job_notebook, job_file)
except Exception as e:

82
recipes/python_recipe.py
View File

@ -6,36 +6,34 @@ along with an appropriate handler for said events.
Author(s): David Marchant
"""
import os
import itertools
import nbformat
import sys
from typing import Any, Tuple
from core.correctness.validation import check_type, valid_string, \
from core.correctness.validation import check_script, valid_string, \
valid_dict, valid_event, valid_existing_dir_path, setup_debugging
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \
EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE
PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
get_job_file, get_base_file, get_result_file
from core.functionality import print_debug, create_job, replace_keywords, \
make_dir, write_yaml, write_notebook
make_dir, write_yaml, write_file, lines_to_string, read_file_lines
from core.meow import BaseRecipe, BaseHandler
class PythonRecipe(BaseRecipe):
def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={},
def __init__(self, name:str, recipe:list[str], parameters:dict[str,Any]={},
requirements:dict[str,Any]={}):
"""PythonRecipe Constructor. This is used to execute python analysis
code."""
super().__init__(name, recipe, parameters, requirements)
def _is_valid_recipe(self, recipe:dict[str,Any])->None:
def _is_valid_recipe(self, recipe:list[str])->None:
"""Validation check for 'recipe' variable from main constructor.
Called within parent BaseRecipe constructor."""
check_type(recipe, dict)
nbformat.validate(recipe)
check_script(recipe)
def _is_valid_parameters(self, parameters:dict[str,Any])->None:
"""Validation check for 'parameters' variable from main constructor.
@ -135,7 +133,7 @@ class PythonHandler(BaseHandler):
extras={
JOB_PARAMETERS:yaml_dict,
JOB_HASH: event[WATCHDOG_HASH],
PYTHON_FUNC:job_func,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:self.output_dir,
PYTHON_EXECUTION_BASE:self.handler_base
}
@ -161,9 +159,9 @@ class PythonHandler(BaseHandler):
meta_file = os.path.join(job_dir, META_FILE)
write_yaml(meow_job, meta_file)
# write an executable notebook to the job directory
base_file = os.path.join(job_dir, BASE_FILE)
write_notebook(event[EVENT_RULE].recipe.recipe, base_file)
# write an executable script to the job directory
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_file(lines_to_string(event[EVENT_RULE].recipe.recipe), base_file)
# write a parameter file to the job directory
param_file = os.path.join(job_dir, PARAMS_FILE)
@ -178,25 +176,26 @@ class PythonHandler(BaseHandler):
self.to_runner.send(job_dir)
# Papermill job execution code, to be run within the conductor
def job_func(job):
def python_job_func(job):
# Requires own imports as will be run in its own execution environment
import sys
import os
import papermill
from datetime import datetime
from core.functionality import write_yaml, read_yaml, write_notebook, \
get_file_hash, parameterize_jupyter_notebook
from core.correctness.vars import JOB_EVENT, EVENT_RULE, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE, \
from io import StringIO
from core.functionality import write_yaml, read_yaml, \
get_file_hash, parameterize_python_script
from core.correctness.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE
event = job[JOB_EVENT]
JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE, get_base_file, \
get_job_file, get_result_file
# Identify job files
job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID])
meta_file = os.path.join(job_dir, META_FILE)
job_file = os.path.join(job_dir, JOB_FILE)
result_file = os.path.join(job_dir, RESULT_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON))
result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON))
param_file = os.path.join(job_dir, PARAMS_FILE)
yaml_dict = read_yaml(param_file)
@ -221,12 +220,13 @@ def job_func(job):
write_yaml(job, meta_file)
return
# Create a parameterised version of the executable notebook
# Create a parameterised version of the executable script
try:
job_notebook = parameterize_jupyter_notebook(
event[EVENT_RULE].recipe.recipe, yaml_dict
base_script = read_file_lines(base_file)
job_script = parameterize_python_script(
base_script, yaml_dict
)
write_notebook(job_notebook, job_file)
write_file(lines_to_string(job_script), job_file)
except Exception as e:
job[JOB_STATUS] = STATUS_FAILED
job[JOB_END_TIME] = datetime.now()
@ -235,13 +235,33 @@ def job_func(job):
write_yaml(job, meta_file)
return
# Execute the parameterised notebook
# Execute the parameterised script
std_stdout = sys.stdout
std_stderr = sys.stderr
try:
papermill.execute_notebook(job_file, result_file, {})
redirected_output = sys.stdout
redirected_error = sys.stderr
exec(open(job_file).read())
write_file(f"""--STDOUT--
{redirected_output}
--STDERR--
{redirected_error}
""",
result_file)
except Exception as e:
sys.stdout = std_stdout
sys.stderr = std_stderr
job[JOB_STATUS] = STATUS_FAILED
job[JOB_END_TIME] = datetime.now()
msg = f"Result file {result_file} was not created successfully. {e}"
job[JOB_ERROR] = msg
write_yaml(job, meta_file)
return
sys.stdout = std_stdout
sys.stderr = std_stderr

1
rules/__init__.py
View File

@ -1,2 +1,3 @@
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from rules.file_event_python_rule import FileEventPythonRule

30
rules/file_event_python_rule.py Normal file
View File

@ -0,0 +1,30 @@
"""
This file contains definitions for a MEOW rule connecting the FileEventPattern
and PythonRecipe.
Author(s): David Marchant
"""
from core.correctness.validation import check_type
from core.meow import BaseRule
from patterns.file_event_pattern import FileEventPattern
from recipes.python_recipe import PythonRecipe
# TODO potentailly remove this and just invoke BaseRule directly, as does not
# add any functionality other than some validation.
class FileEventPythonRule(BaseRule):
pattern_type = "FileEventPattern"
recipe_type = "PythonRecipe"
def __init__(self, name: str, pattern:FileEventPattern,
recipe:PythonRecipe):
super().__init__(name, pattern, recipe)
def _is_valid_pattern(self, pattern:FileEventPattern)->None:
"""Validation check for 'pattern' variable from main constructor. Is
automatically called during initialisation."""
check_type(pattern, FileEventPattern)
def _is_valid_recipe(self, recipe:PythonRecipe)->None:
"""Validation check for 'recipe' variable from main constructor. Is
automatically called during initialisation."""
check_type(recipe, PythonRecipe)

25
tests/shared.py
View File

@ -22,6 +22,31 @@ def teardown():
rmtree(TEST_JOB_OUTPUT)
rmtree("first")
# Recipe funcs
BAREBONES_PYTHON_SCRIPT = [
""
]
COMPLETE_PYTHON_SCRIPT = [
"# Setup parameters",
"num = 1000",
"infile = 'somehere/particular'",
"outfile = 'nowhere/particular'",
"",
"with open(infile, 'r') as file:",
" s = int(file.read())",
""
"for i in range(num):",
" s += i",
"",
"div_by = 4",
"result = s / div_by",
"",
"print(result)",
"",
"with open(outfile, 'w') as file:",
" file.write(str(result))"
]
# Jupyter notebooks
BAREBONES_NOTEBOOK = {
"cells": [],

154
tests/test_conductors.py
View File

@ -4,15 +4,20 @@ import unittest
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \
META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
STATUS_DONE, JOB_TYPE_PAPERMILL, get_base_file, get_result_file, \
get_job_file
from core.functionality import get_file_hash, create_watchdog_event, \
create_job, make_dir, write_yaml, write_notebook
create_job, make_dir, write_yaml, write_notebook, read_yaml, write_file, \
lines_to_string
from core.meow import create_rule
from conductors import LocalPythonConductor
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from recipes.python_recipe import PythonRecipe, python_job_func
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
TEST_JOB_OUTPUT, TEST_HANDLER_BASE
TEST_JOB_OUTPUT, TEST_HANDLER_BASE, COMPLETE_PYTHON_SCRIPT
def failing_func():
@ -32,10 +37,94 @@ class MeowTests(unittest.TestCase):
def testLocalPythonConductorCreation(self)->None:
LocalPythonConductor()
#TODO Test LocalPythonConductor execution criteria
#TODO Test LocalPythonConductor executes valid python jobs
def testLocalPythonConductorValidPythonJob(self)->None:
lpc = LocalPythonConductor()
# Test LocalPythonConductor executes valid jobs
def testLocalPythonConductorValidJob(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
with open(file_path, "w") as f:
f.write("150")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"num":450,
"outfile":result_path
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
rule = create_rule(pattern, recipe)
params_dict = {
"num":450,
"infile":file_path,
"outfile":result_path
}
job_dict = create_job(
JOB_TYPE_PYTHON,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
make_dir(job_dir)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
lpc.execute(job_dict)
self.assertFalse(os.path.exists(job_dir))
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor executes valid papermill jobs
def testLocalPythonConductorValidPapermillJob(self)->None:
lpc = LocalPythonConductor()
file_path = os.path.join(TEST_MONITOR_BASE, "test")
@ -67,7 +156,7 @@ class MeowTests(unittest.TestCase):
}
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -77,7 +166,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -89,7 +178,10 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
meta_path = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_path)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(job_dict)
@ -99,11 +191,22 @@ class MeowTests(unittest.TestCase):
output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
meta_path = os.path.join(output_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], STATUS_DONE)
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
@ -140,7 +243,7 @@ class MeowTests(unittest.TestCase):
}
bad_job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -150,7 +253,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
}
)
@ -160,7 +263,7 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
with self.assertRaises(KeyError):
@ -168,7 +271,7 @@ class MeowTests(unittest.TestCase):
# Ensure execution can continue after one failed job
good_job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -178,7 +281,7 @@ class MeowTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -190,7 +293,7 @@ class MeowTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
lpc.execute(good_job_dict)
@ -201,10 +304,13 @@ class MeowTests(unittest.TestCase):
output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
self.assertTrue(os.path.exists(output_dir))
self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
@ -235,7 +341,7 @@ class MeowTests(unittest.TestCase):
rule = create_rule(pattern, recipe)
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,

13
tests/test_functionality.py
View File

@ -12,7 +12,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
make_dir, parameterize_jupyter_notebook, create_event, create_job, \
replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \
@ -240,6 +240,8 @@ class CorrectnessTests(unittest.TestCase):
pn["cells"][0]["source"],
"# The first cell\n\ns = 4\nnum = 1000")
# TODO Test that parameterize_python_script parameterises given script
# Test that create_event produces valid event dictionary
def testCreateEvent(self)->None:
pattern = FileEventPattern(
@ -307,7 +309,7 @@ class CorrectnessTests(unittest.TestCase):
)
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
event,
extras={
JOB_PARAMETERS:{
@ -328,7 +330,7 @@ class CorrectnessTests(unittest.TestCase):
self.assertIn(JOB_EVENT, job_dict)
self.assertEqual(job_dict[JOB_EVENT], event)
self.assertIn(JOB_TYPE, job_dict)
self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PYTHON)
self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PAPERMILL)
self.assertIn(JOB_PATTERN, job_dict)
self.assertEqual(job_dict[JOB_PATTERN], pattern.name)
self.assertIn(JOB_RECIPE, job_dict)
@ -659,3 +661,8 @@ class CorrectnessTests(unittest.TestCase):
self.assertEqual(event[EVENT_RULE], rule)
self.assertEqual(event["a"], 1)
self.assertEqual(event[WATCHDOG_BASE], "base")
#TODO test read file
#TODO test readlines file
#TODO test write file
#TODO test lines to str

77
tests/test_meow.py
View File

@ -3,6 +3,7 @@ import unittest
from typing import Any, Union, Tuple
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, BaseConductor, create_rules, create_rule
from patterns import FileEventPattern
@ -70,6 +71,78 @@ class MeowTests(unittest.TestCase):
pass
FullPattern("name", "", "", "", "")
# Test expansion of parameter sweeps
def testBasePatternExpandSweeps(self)->None:
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={
"s1":{
SWEEP_START: 10, SWEEP_STOP: 20, SWEEP_JUMP:5
}
})
es = pattern_one.expand_sweeps()
self.assertIsInstance(es, list)
self.assertEqual(len(es), 3)
values = [
"s1-10", "s1-15", "s1-20",
]
for sweep_vals in es:
self.assertIsInstance(sweep_vals, tuple)
self.assertEqual(len(sweep_vals), 1)
val1 = None
for sweep_val in sweep_vals:
self.assertIsInstance(sweep_val, tuple)
self.assertEqual(len(sweep_val), 2)
if sweep_val[0] == "s1":
val1 = f"s1-{sweep_val[1]}"
if val1:
values.remove(val1)
self.assertEqual(len(values), 0)
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={
"s1":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
},
"s2":{
SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
}
})
es = pattern_one.expand_sweeps()
self.assertIsInstance(es, list)
self.assertEqual(len(es), 15)
values = [
"s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20",
"s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35",
"s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50",
"s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65",
"s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80",
]
for sweep_vals in es:
self.assertIsInstance(sweep_vals, tuple)
self.assertEqual(len(sweep_vals), 2)
val1 = None
val2 = None
for sweep_val in sweep_vals:
self.assertIsInstance(sweep_val, tuple)
self.assertEqual(len(sweep_val), 2)
if sweep_val[0] == "s1":
val1 = f"s1-{sweep_val[1]}"
if sweep_val[0] == "s2":
val2 = f"s2-{sweep_val[1]}"
if val1 and val2:
values.remove(f"{val1}/{val2}")
self.assertEqual(len(values), 0)
# Test that BaseRecipe instantiation
def testBaseRule(self)->None:
with self.assertRaises(TypeError):
@ -87,7 +160,7 @@ class MeowTests(unittest.TestCase):
pass
def _is_valid_pattern(self, pattern:Any)->None:
pass
FullRule("name", "", "")
FullRule("name", valid_pattern_one, valid_recipe_one)
# Test that create_rule creates a rule from pattern and recipe
def testCreateRule(self)->None:
@ -227,5 +300,3 @@ class MeowTests(unittest.TestCase):
pass
FullTestConductor()
# TODO Test expansion of parameter sweeps

407
tests/test_recipes.py
View File

@ -7,20 +7,25 @@ from multiprocessing import Pipe
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \
PARAMS_FILE, JOB_FILE, RESULT_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, JOB_ERROR, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from core.correctness.validation import valid_job
from core.functionality import get_file_hash, create_job, \
create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml
create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml, \
write_file, lines_to_string
from core.meow import create_rules, create_rule
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, job_func
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from shared import setup, teardown, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
PapermillHandler, papermill_job_func
from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from rules import FileEventJupyterNotebookRule, FileEventPythonRule
from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
COMPLETE_PYTHON_SCRIPT, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK
class JupyterNotebookTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -349,7 +354,7 @@ class JupyterNotebookTests(unittest.TestCase):
}
job_dict = create_job(
JOB_TYPE_PYTHON,
JOB_TYPE_PAPERMILL,
create_watchdog_event(
file_path,
rule,
@ -359,7 +364,7 @@ class JupyterNotebookTests(unittest.TestCase):
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:job_func,
PYTHON_FUNC:papermill_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
@ -375,25 +380,35 @@ class JupyterNotebookTests(unittest.TestCase):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, BASE_FILE)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
write_notebook(APPENDING_NOTEBOOK, base_file)
job_func(job_dict)
papermill_job_func(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_dir))
self.assertTrue(os.path.exists(os.path.join(job_dir, META_FILE)))
self.assertTrue(os.path.exists(os.path.join(job_dir, BASE_FILE)))
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(os.path.join(job_dir, JOB_FILE)))
self.assertTrue(os.path.exists(os.path.join(job_dir, RESULT_FILE)))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL))))
self.assertTrue(os.path.exists(result_path))
# Test jobFunc doesn't execute with no args
def testJobFuncBadArgs(self)->None:
try:
job_func({})
papermill_job_func({})
except Exception:
pass
@ -411,3 +426,363 @@ class PythonTests(unittest.TestCase):
def tearDown(self)->None:
super().tearDown()
teardown()
# Test PythonRecipe can be created
def testPythonRecipeCreationMinimum(self)->None:
PythonRecipe("test_recipe", BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe cannot be created without name
def testPythonRecipeCreationNoName(self)->None:
with self.assertRaises(ValueError):
PythonRecipe("", BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe cannot be created with invalid name
def testPythonRecipeCreationInvalidName(self)->None:
with self.assertRaises(ValueError):
PythonRecipe("@test_recipe", BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe cannot be created with invalid recipe
def testPythonRecipeCreationInvalidRecipe(self)->None:
with self.assertRaises(TypeError):
PythonRecipe("test_recipe", BAREBONES_NOTEBOOK)
# Test PythonRecipe name setup correctly
def testPythonRecipeSetupName(self)->None:
name = "name"
pr = PythonRecipe(name, BAREBONES_PYTHON_SCRIPT)
self.assertEqual(pr.name, name)
# Test PythonRecipe recipe setup correctly
def testPythonRecipeSetupRecipe(self)->None:
pr = PythonRecipe("name", BAREBONES_PYTHON_SCRIPT)
self.assertEqual(pr.recipe, BAREBONES_PYTHON_SCRIPT)
# Test PythonRecipe parameters setup correctly
def testPythonRecipeSetupParameters(self)->None:
parameters = {
"a": 1,
"b": True
}
pr = PythonRecipe(
"name", BAREBONES_PYTHON_SCRIPT, parameters=parameters)
self.assertEqual(pr.parameters, parameters)
# Test PythonRecipe requirements setup correctly
def testPythonRecipeSetupRequirements(self)->None:
requirements = {
"a": 1,
"b": True
}
pr = PythonRecipe(
"name", BAREBONES_PYTHON_SCRIPT, requirements=requirements)
self.assertEqual(pr.requirements, requirements)
# Test PythonHandler can be created
def testPythonHandlerMinimum(self)->None:
PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
# Test PythonHandler will handle given events
def testPythonHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
if from_handler_reader.poll(3):
job_dir = from_handler_reader.recv()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
# Test PythonHandler will create enough jobs from single sweep
def testPythonHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={"s":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
}})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
else:
recieving = False
values = [0, 1, 2]
self.assertEqual(len(jobs), 3)
for job_dir in jobs:
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
self.assertIn(JOB_PARAMETERS, job)
self.assertIn("s", job[JOB_PARAMETERS])
if job[JOB_PARAMETERS]["s"] in values:
values.remove(job[JOB_PARAMETERS]["s"])
self.assertEqual(len(values), 0)
# Test PythonHandler will create enough jobs from multiple sweeps
def testPythonHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = PythonHandler(
TEST_HANDLER_BASE,
TEST_JOB_OUTPUT
)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={
"s1":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
},
"s2":{
SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
}
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
else:
recieving = False
values = [
"s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20",
"s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35",
"s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50",
"s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65",
"s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80",
]
self.assertEqual(len(jobs), 15)
for job_dir in jobs:
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
self.assertIn(JOB_PARAMETERS, job)
val1 = None
val2 = None
if "s1" in job[JOB_PARAMETERS]:
val1 = f"s1-{job[JOB_PARAMETERS]['s1']}"
if "s2" in job[JOB_PARAMETERS]:
val2 = f"s2-{job[JOB_PARAMETERS]['s2']}"
val = None
if val1 and val2:
val = f"{val1}/{val2}"
if val and val in values:
values.remove(val)
self.assertEqual(len(values), 0)
# Test jobFunc performs as expected
def testJobFunc(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
with open(file_path, "w") as f:
f.write("250")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile": result_path
})
recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT)
rule = create_rule(pattern, recipe)
params_dict = {
"extra":"extra",
"infile":file_path,
"outfile": result_path
}
job_dict = create_job(
JOB_TYPE_PYTHON,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash,
PYTHON_FUNC:python_job_func,
PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
}
)
job_dir = os.path.join(
job_dict[PYTHON_EXECUTION_BASE], job_dict[JOB_ID])
make_dir(job_dir)
meta_file = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_file)
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(params_dict, param_file)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
write_notebook(APPENDING_NOTEBOOK, base_file)
write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
python_job_func(job_dict)
job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
self.assertTrue(os.path.exists(job_dir))
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE)))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON))))
self.assertTrue(os.path.exists(result_path))
# Test jobFunc doesn't execute with no args
def testJobFuncBadArgs(self)->None:
try:
python_job_func({})
except Exception:
pass
self.assertEqual(len(os.listdir(TEST_HANDLER_BASE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
# TODO test default parameter function execution

15
tests/test_runner.py
View File

@ -6,7 +6,7 @@ import unittest
from time import sleep
from conductors import LocalPythonConductor
from core.correctness.vars import RESULT_FILE
from core.correctness.vars import get_result_file, JOB_TYPE_PAPERMILL
from core.functionality import make_dir, read_notebook
from core.meow import BaseMonitor, BaseHandler, BaseConductor
from core.runner import MeowRunner
@ -187,7 +187,8 @@ class MeowTests(unittest.TestCase):
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(job_dir)), 5)
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
result = read_notebook(
os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
self.assertIsNotNone(result)
output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
@ -279,7 +280,8 @@ class MeowTests(unittest.TestCase):
mid_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(mid_job_dir)), 5)
result = read_notebook(os.path.join(mid_job_dir, RESULT_FILE))
result = read_notebook(
os.path.join(mid_job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
self.assertIsNotNone(result)
mid_output_path = os.path.join(TEST_MONITOR_BASE, "middle", "A.txt")
@ -293,7 +295,8 @@ class MeowTests(unittest.TestCase):
final_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(final_job_dir)), 5)
result = read_notebook(os.path.join(final_job_dir, RESULT_FILE))
result = read_notebook(os.path.join(final_job_dir,
get_result_file(JOB_TYPE_PAPERMILL)))
self.assertIsNotNone(result)
final_output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
@ -305,8 +308,8 @@ class MeowTests(unittest.TestCase):
self.assertEqual(data,
"Initial Data\nA line from Pattern 1\nA line from Pattern 2")
# TODO sweep tests
# TODO adding tests with numpy
# TODO sweep execution test
# TODO adding tests with numpy or other external dependency
# TODO test getting job cannot handle
# TODO test getting event cannot handle
# TODO test with several matched monitors

11
tests/test_validation.py
View File

@ -9,7 +9,7 @@ from typing import Any, Union
from core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
setup_debugging, valid_watchdog_event
setup_debugging, valid_watchdog_event, check_callable
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
@ -304,7 +304,7 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(TypeError):
setup_debugging(stream, "1")
#Test watchdog event dict
# Test watchdog event dict
def testWatchdogEventValidation(self)->None:
valid_watchdog_event({
EVENT_TYPE: "test",
@ -337,3 +337,10 @@ class CorrectnessTests(unittest.TestCase):
with self.assertRaises(KeyError):
valid_event({})
# Test check_callable
def testCheckCallable(self)->None:
check_callable(make_dir)
with self.assertRaises(TypeError):
check_callable("a")