From 47f9fe73fa1a5b1173342a4696ee35049b1f58d6 Mon Sep 17 00:00:00 2001 From: PatchOfScotland Date: Fri, 3 Feb 2023 14:47:16 +0100 Subject: [PATCH] differentiated papermill and python jobs more clearly --- conductors/local_python_conductor.py | 4 +- core/correctness/validation.py | 14 + core/correctness/vars.py | 26 +- core/functionality.py | 53 +++- core/meow.py | 14 +- patterns/file_event_pattern.py | 4 +- recipes/jupyter_notebook_recipe.py | 38 +-- recipes/python_recipe.py | 82 ++++-- rules/__init__.py | 1 + rules/file_event_python_rule.py | 30 ++ tests/shared.py | 27 +- tests/test_conductors.py | 154 ++++++++-- tests/test_functionality.py | 13 +- tests/test_meow.py | 77 ++++- tests/test_recipes.py | 407 +++++++++++++++++++++++++-- tests/test_runner.py | 15 +- tests/test_validation.py | 13 +- 17 files changed, 853 insertions(+), 119 deletions(-) create mode 100644 rules/file_event_python_rule.py diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index 7651584..34522a0 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -14,7 +14,7 @@ from typing import Any, Tuple from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \ STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, PYTHON_OUTPUT_DIR, \ - JOB_TYPE + JOB_TYPE, JOB_TYPE_PAPERMILL from core.correctness.validation import valid_job from core.functionality import read_yaml, write_yaml from core.meow import BaseConductor @@ -30,7 +30,7 @@ class LocalPythonConductor(BaseConductor): process it or not. This conductor will accept any Python job type""" try: valid_job(job) - if job[JOB_TYPE] == JOB_TYPE_PYTHON: + if job[JOB_TYPE] in [JOB_TYPE_PYTHON, JOB_TYPE_PAPERMILL]: return True, "" except Exception as e: pass diff --git a/core/correctness/validation.py b/core/correctness/validation.py index bca3135..e6585b6 100644 --- a/core/correctness/validation.py +++ b/core/correctness/validation.py @@ -72,6 +72,12 @@ def check_type(variable:Any, expected_type:type, alt_types:list[type]=[], % (get_args(expected_type), type(variable)) ) +def check_callable(call:Any)->None: + """Checks if a given variable is a callable function. Raises TypeError if + not.""" + if not callable(call): + raise TypeError(f"Given object '{call}' is not a callable function") + def check_implementation(child_func, parent_class): """Checks if the given function has been overridden from the one inherited from the parent class. Raises a NotImplementedError if this is the case.""" @@ -94,6 +100,14 @@ def check_implementation(child_func, parent_class): msg = get_not_imp_msg(parent_class, parent_func) raise NotImplementedError(msg) +def check_script(script:Any): + """Checks if a given variable is a valid script. Raises TypeError if + not.""" + # TODO investigate more robust check here + check_type(script, list) + for line in script: + check_type(line, str) + def valid_string(variable:str, valid_chars:str, min_length:int=1)->None: """Checks that all characters in a given string are present in a provided list of characters. Will raise an ValueError if unexpected character is diff --git a/core/correctness/vars.py b/core/correctness/vars.py index 103fc4d..bf46754 100644 --- a/core/correctness/vars.py +++ b/core/correctness/vars.py @@ -80,10 +80,24 @@ DIR_EVENTS = [ # meow jobs JOB_TYPE = "job_type" JOB_TYPE_PYTHON = "python" +JOB_TYPE_PAPERMILL = "papermill" PYTHON_FUNC = "func" PYTHON_EXECUTION_BASE = "exection_base" PYTHON_OUTPUT_DIR = "output_dir" +JOB_TYPES = { + JOB_TYPE_PAPERMILL: [ + "base.ipynb", + "job.ipynb", + "result.ipynb", + ], + JOB_TYPE_PYTHON: [ + "base.py", + "job.py", + "result.py", + ] +} + # job definitions JOB_ID = "id" JOB_EVENT = "event" @@ -108,10 +122,7 @@ STATUS_DONE = "done" # job definition files META_FILE = "job.yml" -BASE_FILE = "base.ipynb" PARAMS_FILE = "params.yml" -JOB_FILE = "job.ipynb" -RESULT_FILE = "result.ipynb" # Parameter sweep keys SWEEP_START = "start" @@ -132,3 +143,12 @@ def get_not_imp_msg(parent_class, class_function): return f"Children of the '{parent_class.__name__}' class must implement " \ f"the '{class_function.__name__}({signature(class_function)})' " \ "function" + +def get_base_file(job_type:str): + return JOB_TYPES[job_type][0] + +def get_job_file(job_type:str): + return JOB_TYPES[job_type][1] + +def get_result_file(job_type:str): + return JOB_TYPES[job_type][2] diff --git a/core/functionality.py b/core/functionality.py index dd1bcbb..126b9c4 100644 --- a/core/functionality.py +++ b/core/functionality.py @@ -1,4 +1,4 @@ - +# TODO comments import copy import hashlib import json @@ -15,7 +15,7 @@ from typing import Any from random import SystemRandom from core.correctness.validation import check_type, valid_existing_file_path, \ - valid_path + valid_path, check_script from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ VALID_CHANNELS, HASH_BUFFER_SIZE, SHA256, DEBUG_WARNING, DEBUG_INFO, \ EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \ @@ -128,6 +128,18 @@ def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False): os.makedirs(path, exist_ok=can_exist) +def read_file(filepath:str): + with open(filepath, 'r') as file: + return file.read() + +def read_file_lines(filepath:str): + with open(filepath, 'r') as file: + return file.readlines() + +def write_file(source:str, filename:str): + with open(filename, 'w') as file: + file.write(source) + def read_yaml(filepath:str): """ Reads a file path as a yaml object. @@ -171,7 +183,7 @@ def write_notebook(source:dict[str,Any], filename:str): json.dump(source, job_file) # Adapted from: https://github.com/rasmunk/notebook_parameterizer -def parameterize_jupyter_notebook( jupyter_notebook:dict[str,Any], +def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any], parameters:dict[str,Any], expand_env_values:bool=False)->dict[str,Any]: nbformat.validate(jupyter_notebook) check_type(parameters, dict) @@ -244,6 +256,38 @@ def parameterize_jupyter_notebook( jupyter_notebook:dict[str,Any], return output_notebook +def parameterize_python_script(script:list[str], parameters:dict[str,Any], + expand_env_values:bool=False)->dict[str,Any]: + check_script(script) + check_type(parameters, dict) + + output_script = copy.deepcopy(script) + + for i, line in enumerate(output_script): + if "=" in line: + d_line = list(map(lambda x: x.replace(" ", ""), + line.split("="))) + # Matching parameter name + if len(d_line) == 2 and d_line[0] in parameters: + value = parameters[d_line[0]] + # Whether to expand value from os env + if ( + expand_env_values + and isinstance(value, str) + and value.startswith("ENV_") + ): + env_var = value.replace("ENV_", "") + value = os.getenv( + env_var, + "MISSING ENVIRONMENT VARIABLE: {}".format(env_var) + ) + output_script[i] = f"{d_line[0]} = {repr(value)}" + + # Validate that the parameterized notebook is still valid + check_script(output_script) + + return output_script + def print_debug(print_target, debug_level, msg, level)->None: if print_target is None: return @@ -338,3 +382,6 @@ def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={} } return {**extras, **job_dict} + +def lines_to_string(lines:list[str])->str: + return "\n".join(lines) diff --git a/core/meow.py b/core/meow.py index fd38b32..b4287e9 100644 --- a/core/meow.py +++ b/core/meow.py @@ -98,7 +98,6 @@ class BasePattern: check_implementation(type(self)._is_valid_recipe, BasePattern) check_implementation(type(self)._is_valid_parameters, BasePattern) check_implementation(type(self)._is_valid_output, BasePattern) - check_implementation(type(self)._is_valid_sweep, BasePattern) self._is_valid_name(name) self.name = name self._is_valid_recipe(recipe) @@ -140,8 +139,9 @@ class BasePattern: pass def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None: - """Validation check for 'sweep' variable from main constructor. Must - be implemented by any child class.""" + """Validation check for 'sweep' variable from main constructor. This + function is implemented to check for the types given in the signature, + and must be overridden if these differ.""" check_type(sweep, dict) if not sweep: return @@ -208,13 +208,19 @@ class BaseRule: the input parameters.""" check_implementation(type(self)._is_valid_pattern, BaseRule) check_implementation(type(self)._is_valid_recipe, BaseRule) + self.__check_types_set() self._is_valid_name(name) self.name = name self._is_valid_pattern(pattern) self.pattern = pattern self._is_valid_recipe(recipe) self.recipe = recipe - self.__check_types_set() + check_type(pattern, BasePattern) + check_type(recipe, BaseRecipe) + if pattern.recipe != recipe.name: + raise ValueError(f"Cannot create Rule {name}. Pattern " + f"{pattern.name} does not identify Recipe {recipe.name}. It " + f"uses {pattern.recipe}") def __new__(cls, *args, **kwargs): """A check that this base class is not instantiated itself, only diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 67aa43e..d7a629c 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -23,8 +23,8 @@ from core.correctness.validation import check_type, valid_string, \ setup_debugging from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \ - FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \ - WATCHDOG_BASE, FILE_RETROACTIVE_EVENT, WATCHDOG_HASH, SHA256 + FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \ + FILE_RETROACTIVE_EVENT, SHA256 from core.functionality import print_debug, create_watchdog_event, \ get_file_hash, create_fake_watchdog_event from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \ diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index 3f934d9..db82ff3 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -6,7 +6,6 @@ notebooks, along with an appropriate handler for said events. Author(s): David Marchant """ import os -import itertools import nbformat import sys @@ -17,12 +16,12 @@ from core.correctness.validation import check_type, valid_string, \ valid_event from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \ - EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \ - PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \ + EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \ + PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, \ PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \ - EVENT_RULE + EVENT_RULE, get_base_file, get_job_file, get_result_file from core.functionality import print_debug, create_job, replace_keywords, \ - make_dir, write_yaml, write_notebook + make_dir, write_yaml, write_notebook, read_notebook from core.meow import BaseRecipe, BaseHandler @@ -140,19 +139,19 @@ class PapermillHandler(BaseHandler): """Function to set up new job dict and send it to the runner to be executed.""" meow_job = create_job( - JOB_TYPE_PYTHON, + JOB_TYPE_PAPERMILL, event, extras={ JOB_PARAMETERS:yaml_dict, JOB_HASH: event[WATCHDOG_HASH], - PYTHON_FUNC:job_func, + PYTHON_FUNC:papermill_job_func, PYTHON_OUTPUT_DIR:self.output_dir, PYTHON_EXECUTION_BASE:self.handler_base } ) print_debug(self._print_target, self.debug_level, f"Creating job from event at {event[EVENT_PATH]} of type " - f"{JOB_TYPE_PYTHON}.", DEBUG_INFO) + f"{JOB_TYPE_PAPERMILL}.", DEBUG_INFO) # replace MEOW keyworks within variables dict yaml_dict = replace_keywords( @@ -172,7 +171,7 @@ class PapermillHandler(BaseHandler): write_yaml(meow_job, meta_file) # write an executable notebook to the job directory - base_file = os.path.join(job_dir, BASE_FILE) + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL)) write_notebook(event[EVENT_RULE].recipe.recipe, base_file) # write a parameter file to the job directory @@ -188,25 +187,26 @@ class PapermillHandler(BaseHandler): self.to_runner.send(job_dir) # Papermill job execution code, to be run within the conductor -def job_func(job): +def papermill_job_func(job): # Requires own imports as will be run in its own execution environment import os import papermill from datetime import datetime from core.functionality import write_yaml, read_yaml, write_notebook, \ get_file_hash, parameterize_jupyter_notebook - from core.correctness.vars import JOB_EVENT, EVENT_RULE, JOB_ID, \ - EVENT_PATH, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE, \ + from core.correctness.vars import JOB_EVENT, JOB_ID, \ + EVENT_PATH, META_FILE, PARAMS_FILE, \ JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \ - JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE - - event = job[JOB_EVENT] + JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE, get_job_file, \ + get_result_file # Identify job files job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID]) meta_file = os.path.join(job_dir, META_FILE) - job_file = os.path.join(job_dir, JOB_FILE) - result_file = os.path.join(job_dir, RESULT_FILE) + # TODO fix these paths so they are dynamic + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL)) + job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL)) + result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL)) param_file = os.path.join(job_dir, PARAMS_FILE) yaml_dict = read_yaml(param_file) @@ -233,8 +233,10 @@ def job_func(job): # Create a parameterised version of the executable notebook try: + base_notebook = read_notebook(base_file) + # TODO read notebook from already written file rather than event job_notebook = parameterize_jupyter_notebook( - event[EVENT_RULE].recipe.recipe, yaml_dict + base_notebook, yaml_dict ) write_notebook(job_notebook, job_file) except Exception as e: diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index 7e9b613..59d344b 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -6,36 +6,34 @@ along with an appropriate handler for said events. Author(s): David Marchant """ import os -import itertools -import nbformat import sys from typing import Any, Tuple -from core.correctness.validation import check_type, valid_string, \ +from core.correctness.validation import check_script, valid_string, \ valid_dict, valid_event, valid_existing_dir_path, setup_debugging from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \ EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \ - PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \ - PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE + PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, \ + PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \ + get_job_file, get_base_file, get_result_file from core.functionality import print_debug, create_job, replace_keywords, \ - make_dir, write_yaml, write_notebook + make_dir, write_yaml, write_file, lines_to_string, read_file_lines from core.meow import BaseRecipe, BaseHandler class PythonRecipe(BaseRecipe): - def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={}, + def __init__(self, name:str, recipe:list[str], parameters:dict[str,Any]={}, requirements:dict[str,Any]={}): """PythonRecipe Constructor. This is used to execute python analysis code.""" super().__init__(name, recipe, parameters, requirements) - def _is_valid_recipe(self, recipe:dict[str,Any])->None: + def _is_valid_recipe(self, recipe:list[str])->None: """Validation check for 'recipe' variable from main constructor. Called within parent BaseRecipe constructor.""" - check_type(recipe, dict) - nbformat.validate(recipe) + check_script(recipe) def _is_valid_parameters(self, parameters:dict[str,Any])->None: """Validation check for 'parameters' variable from main constructor. @@ -135,7 +133,7 @@ class PythonHandler(BaseHandler): extras={ JOB_PARAMETERS:yaml_dict, JOB_HASH: event[WATCHDOG_HASH], - PYTHON_FUNC:job_func, + PYTHON_FUNC:python_job_func, PYTHON_OUTPUT_DIR:self.output_dir, PYTHON_EXECUTION_BASE:self.handler_base } @@ -161,9 +159,9 @@ class PythonHandler(BaseHandler): meta_file = os.path.join(job_dir, META_FILE) write_yaml(meow_job, meta_file) - # write an executable notebook to the job directory - base_file = os.path.join(job_dir, BASE_FILE) - write_notebook(event[EVENT_RULE].recipe.recipe, base_file) + # write an executable script to the job directory + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON)) + write_file(lines_to_string(event[EVENT_RULE].recipe.recipe), base_file) # write a parameter file to the job directory param_file = os.path.join(job_dir, PARAMS_FILE) @@ -178,25 +176,26 @@ class PythonHandler(BaseHandler): self.to_runner.send(job_dir) # Papermill job execution code, to be run within the conductor -def job_func(job): +def python_job_func(job): # Requires own imports as will be run in its own execution environment + import sys import os - import papermill from datetime import datetime - from core.functionality import write_yaml, read_yaml, write_notebook, \ - get_file_hash, parameterize_jupyter_notebook - from core.correctness.vars import JOB_EVENT, EVENT_RULE, JOB_ID, \ - EVENT_PATH, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE, \ + from io import StringIO + from core.functionality import write_yaml, read_yaml, \ + get_file_hash, parameterize_python_script + from core.correctness.vars import JOB_EVENT, JOB_ID, \ + EVENT_PATH, META_FILE, PARAMS_FILE, \ JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \ - JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE - - event = job[JOB_EVENT] + JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE, get_base_file, \ + get_job_file, get_result_file # Identify job files job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID]) meta_file = os.path.join(job_dir, META_FILE) - job_file = os.path.join(job_dir, JOB_FILE) - result_file = os.path.join(job_dir, RESULT_FILE) + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON)) + job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON)) + result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON)) param_file = os.path.join(job_dir, PARAMS_FILE) yaml_dict = read_yaml(param_file) @@ -221,12 +220,13 @@ def job_func(job): write_yaml(job, meta_file) return - # Create a parameterised version of the executable notebook + # Create a parameterised version of the executable script try: - job_notebook = parameterize_jupyter_notebook( - event[EVENT_RULE].recipe.recipe, yaml_dict + base_script = read_file_lines(base_file) + job_script = parameterize_python_script( + base_script, yaml_dict ) - write_notebook(job_notebook, job_file) + write_file(lines_to_string(job_script), job_file) except Exception as e: job[JOB_STATUS] = STATUS_FAILED job[JOB_END_TIME] = datetime.now() @@ -235,13 +235,33 @@ def job_func(job): write_yaml(job, meta_file) return - # Execute the parameterised notebook + # Execute the parameterised script + std_stdout = sys.stdout + std_stderr = sys.stderr try: - papermill.execute_notebook(job_file, result_file, {}) + redirected_output = sys.stdout + redirected_error = sys.stderr + + exec(open(job_file).read()) + + write_file(f"""--STDOUT-- + {redirected_output} + + --STDERR-- + {redirected_error} + """, + result_file) + except Exception as e: + sys.stdout = std_stdout + sys.stderr = std_stderr + job[JOB_STATUS] = STATUS_FAILED job[JOB_END_TIME] = datetime.now() msg = f"Result file {result_file} was not created successfully. {e}" job[JOB_ERROR] = msg write_yaml(job, meta_file) return + + sys.stdout = std_stdout + sys.stderr = std_stderr diff --git a/rules/__init__.py b/rules/__init__.py index 69081c5..73a7d6c 100644 --- a/rules/__init__.py +++ b/rules/__init__.py @@ -1,2 +1,3 @@ from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule +from rules.file_event_python_rule import FileEventPythonRule \ No newline at end of file diff --git a/rules/file_event_python_rule.py b/rules/file_event_python_rule.py new file mode 100644 index 0000000..8711845 --- /dev/null +++ b/rules/file_event_python_rule.py @@ -0,0 +1,30 @@ + +""" +This file contains definitions for a MEOW rule connecting the FileEventPattern +and PythonRecipe. + +Author(s): David Marchant +""" +from core.correctness.validation import check_type +from core.meow import BaseRule +from patterns.file_event_pattern import FileEventPattern +from recipes.python_recipe import PythonRecipe + +# TODO potentailly remove this and just invoke BaseRule directly, as does not +# add any functionality other than some validation. +class FileEventPythonRule(BaseRule): + pattern_type = "FileEventPattern" + recipe_type = "PythonRecipe" + def __init__(self, name: str, pattern:FileEventPattern, + recipe:PythonRecipe): + super().__init__(name, pattern, recipe) + + def _is_valid_pattern(self, pattern:FileEventPattern)->None: + """Validation check for 'pattern' variable from main constructor. Is + automatically called during initialisation.""" + check_type(pattern, FileEventPattern) + + def _is_valid_recipe(self, recipe:PythonRecipe)->None: + """Validation check for 'recipe' variable from main constructor. Is + automatically called during initialisation.""" + check_type(recipe, PythonRecipe) diff --git a/tests/shared.py b/tests/shared.py index 83b51ad..afa4f0a 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -22,6 +22,31 @@ def teardown(): rmtree(TEST_JOB_OUTPUT) rmtree("first") +# Recipe funcs +BAREBONES_PYTHON_SCRIPT = [ + "" +] +COMPLETE_PYTHON_SCRIPT = [ + "# Setup parameters", + "num = 1000", + "infile = 'somehere/particular'", + "outfile = 'nowhere/particular'", + "", + "with open(infile, 'r') as file:", + " s = int(file.read())", + "" + "for i in range(num):", + " s += i", + "", + "div_by = 4", + "result = s / div_by", + "", + "print(result)", + "", + "with open(outfile, 'w') as file:", + " file.write(str(result))" +] + # Jupyter notebooks BAREBONES_NOTEBOOK = { "cells": [], @@ -261,4 +286,4 @@ ADDING_NOTEBOOK = { }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/tests/test_conductors.py b/tests/test_conductors.py index 1be913b..10b693e 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -4,15 +4,20 @@ import unittest from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \ JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \ - META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE + META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \ + STATUS_DONE, JOB_TYPE_PAPERMILL, get_base_file, get_result_file, \ + get_job_file from core.functionality import get_file_hash, create_watchdog_event, \ - create_job, make_dir, write_yaml, write_notebook + create_job, make_dir, write_yaml, write_notebook, read_yaml, write_file, \ + lines_to_string from core.meow import create_rule from conductors import LocalPythonConductor from patterns import FileEventPattern -from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func +from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ + papermill_job_func +from recipes.python_recipe import PythonRecipe, python_job_func from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \ - TEST_JOB_OUTPUT, TEST_HANDLER_BASE + TEST_JOB_OUTPUT, TEST_HANDLER_BASE, COMPLETE_PYTHON_SCRIPT def failing_func(): @@ -32,10 +37,94 @@ class MeowTests(unittest.TestCase): def testLocalPythonConductorCreation(self)->None: LocalPythonConductor() - #TODO Test LocalPythonConductor execution criteria + #TODO Test LocalPythonConductor executes valid python jobs + def testLocalPythonConductorValidPythonJob(self)->None: + lpc = LocalPythonConductor() - # Test LocalPythonConductor executes valid jobs - def testLocalPythonConductorValidJob(self)->None: + file_path = os.path.join(TEST_MONITOR_BASE, "test") + result_path = os.path.join(TEST_MONITOR_BASE, "output") + + with open(file_path, "w") as f: + f.write("150") + + file_hash = get_file_hash(file_path, SHA256) + + pattern = FileEventPattern( + "pattern", + file_path, + "recipe_one", + "infile", + parameters={ + "num":450, + "outfile":result_path + }) + recipe = PythonRecipe( + "recipe_one", COMPLETE_PYTHON_SCRIPT) + + rule = create_rule(pattern, recipe) + + params_dict = { + "num":450, + "infile":file_path, + "outfile":result_path + } + + job_dict = create_job( + JOB_TYPE_PYTHON, + create_watchdog_event( + file_path, + rule, + TEST_MONITOR_BASE, + file_hash + ), + extras={ + JOB_PARAMETERS:params_dict, + JOB_HASH: file_hash, + PYTHON_FUNC:python_job_func, + PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT, + PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE + } + ) + + job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID]) + make_dir(job_dir) + + param_file = os.path.join(job_dir, PARAMS_FILE) + write_yaml(params_dict, param_file) + + meta_path = os.path.join(job_dir, META_FILE) + write_yaml(job_dict, meta_path) + + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON)) + write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file) + + lpc.execute(job_dict) + + self.assertFalse(os.path.exists(job_dir)) + + output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID]) + self.assertTrue(os.path.exists(output_dir)) + + meta_path = os.path.join(output_dir, META_FILE) + self.assertTrue(os.path.exists(meta_path)) + status = read_yaml(meta_path) + self.assertIsInstance(status, dict) + self.assertIn(JOB_STATUS, status) + self.assertEqual(status[JOB_STATUS], STATUS_DONE) + + self.assertNotIn(JOB_ERROR, status) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_base_file(JOB_TYPE_PYTHON)))) + self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE))) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_job_file(JOB_TYPE_PYTHON)))) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_result_file(JOB_TYPE_PYTHON)))) + + self.assertTrue(os.path.exists(result_path)) + + # Test LocalPythonConductor executes valid papermill jobs + def testLocalPythonConductorValidPapermillJob(self)->None: lpc = LocalPythonConductor() file_path = os.path.join(TEST_MONITOR_BASE, "test") @@ -67,7 +156,7 @@ class MeowTests(unittest.TestCase): } job_dict = create_job( - JOB_TYPE_PYTHON, + JOB_TYPE_PAPERMILL, create_watchdog_event( file_path, rule, @@ -77,7 +166,7 @@ class MeowTests(unittest.TestCase): extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, - PYTHON_FUNC:job_func, + PYTHON_FUNC:papermill_job_func, PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT, PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE } @@ -89,7 +178,10 @@ class MeowTests(unittest.TestCase): param_file = os.path.join(job_dir, PARAMS_FILE) write_yaml(params_dict, param_file) - base_file = os.path.join(job_dir, BASE_FILE) + meta_path = os.path.join(job_dir, META_FILE) + write_yaml(job_dict, meta_path) + + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL)) write_notebook(APPENDING_NOTEBOOK, base_file) lpc.execute(job_dict) @@ -99,11 +191,22 @@ class MeowTests(unittest.TestCase): output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID]) self.assertTrue(os.path.exists(output_dir)) - self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE))) - self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE))) + + + meta_path = os.path.join(output_dir, META_FILE) + self.assertTrue(os.path.exists(meta_path)) + status = read_yaml(meta_path) + self.assertIsInstance(status, dict) + self.assertIn(JOB_STATUS, status) + self.assertEqual(status[JOB_STATUS], STATUS_DONE) + + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL)))) self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE))) - self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE))) - self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE))) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL)))) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL)))) self.assertTrue(os.path.exists(result_path)) @@ -140,7 +243,7 @@ class MeowTests(unittest.TestCase): } bad_job_dict = create_job( - JOB_TYPE_PYTHON, + JOB_TYPE_PAPERMILL, create_watchdog_event( file_path, rule, @@ -150,7 +253,7 @@ class MeowTests(unittest.TestCase): extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, - PYTHON_FUNC:job_func, + PYTHON_FUNC:papermill_job_func, } ) @@ -160,7 +263,7 @@ class MeowTests(unittest.TestCase): param_file = os.path.join(job_dir, PARAMS_FILE) write_yaml(params_dict, param_file) - base_file = os.path.join(job_dir, BASE_FILE) + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL)) write_notebook(APPENDING_NOTEBOOK, base_file) with self.assertRaises(KeyError): @@ -168,7 +271,7 @@ class MeowTests(unittest.TestCase): # Ensure execution can continue after one failed job good_job_dict = create_job( - JOB_TYPE_PYTHON, + JOB_TYPE_PAPERMILL, create_watchdog_event( file_path, rule, @@ -178,7 +281,7 @@ class MeowTests(unittest.TestCase): extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, - PYTHON_FUNC:job_func, + PYTHON_FUNC:papermill_job_func, PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT, PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE } @@ -190,7 +293,7 @@ class MeowTests(unittest.TestCase): param_file = os.path.join(job_dir, PARAMS_FILE) write_yaml(params_dict, param_file) - base_file = os.path.join(job_dir, BASE_FILE) + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL)) write_notebook(APPENDING_NOTEBOOK, base_file) lpc.execute(good_job_dict) @@ -201,10 +304,13 @@ class MeowTests(unittest.TestCase): output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID]) self.assertTrue(os.path.exists(output_dir)) self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE))) - self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE))) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL)))) self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE))) - self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE))) - self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE))) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL)))) + self.assertTrue(os.path.exists( + os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL)))) self.assertTrue(os.path.exists(result_path)) @@ -235,7 +341,7 @@ class MeowTests(unittest.TestCase): rule = create_rule(pattern, recipe) job_dict = create_job( - JOB_TYPE_PYTHON, + JOB_TYPE_PAPERMILL, create_watchdog_event( file_path, rule, diff --git a/tests/test_functionality.py b/tests/test_functionality.py index 39df83d..87f72f0 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -12,7 +12,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \ PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, JOB_EVENT, \ JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ - JOB_REQUIREMENTS, STATUS_QUEUED + JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL from core.functionality import generate_id, wait, get_file_hash, rmtree, \ make_dir, parameterize_jupyter_notebook, create_event, create_job, \ replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \ @@ -240,6 +240,8 @@ class CorrectnessTests(unittest.TestCase): pn["cells"][0]["source"], "# The first cell\n\ns = 4\nnum = 1000") + # TODO Test that parameterize_python_script parameterises given script + # Test that create_event produces valid event dictionary def testCreateEvent(self)->None: pattern = FileEventPattern( @@ -307,7 +309,7 @@ class CorrectnessTests(unittest.TestCase): ) job_dict = create_job( - JOB_TYPE_PYTHON, + JOB_TYPE_PAPERMILL, event, extras={ JOB_PARAMETERS:{ @@ -328,7 +330,7 @@ class CorrectnessTests(unittest.TestCase): self.assertIn(JOB_EVENT, job_dict) self.assertEqual(job_dict[JOB_EVENT], event) self.assertIn(JOB_TYPE, job_dict) - self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PYTHON) + self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PAPERMILL) self.assertIn(JOB_PATTERN, job_dict) self.assertEqual(job_dict[JOB_PATTERN], pattern.name) self.assertIn(JOB_RECIPE, job_dict) @@ -659,3 +661,8 @@ class CorrectnessTests(unittest.TestCase): self.assertEqual(event[EVENT_RULE], rule) self.assertEqual(event["a"], 1) self.assertEqual(event[WATCHDOG_BASE], "base") + +#TODO test read file +#TODO test readlines file +#TODO test write file +#TODO test lines to str \ No newline at end of file diff --git a/tests/test_meow.py b/tests/test_meow.py index 081d75b..8c3bc60 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -3,6 +3,7 @@ import unittest from typing import Any, Union, Tuple +from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \ BaseHandler, BaseConductor, create_rules, create_rule from patterns import FileEventPattern @@ -70,6 +71,78 @@ class MeowTests(unittest.TestCase): pass FullPattern("name", "", "", "", "") + # Test expansion of parameter sweeps + def testBasePatternExpandSweeps(self)->None: + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one", sweep={ + "s1":{ + SWEEP_START: 10, SWEEP_STOP: 20, SWEEP_JUMP:5 + } + }) + + es = pattern_one.expand_sweeps() + + self.assertIsInstance(es, list) + self.assertEqual(len(es), 3) + + values = [ + "s1-10", "s1-15", "s1-20", + ] + + for sweep_vals in es: + self.assertIsInstance(sweep_vals, tuple) + self.assertEqual(len(sweep_vals), 1) + + val1 = None + for sweep_val in sweep_vals: + self.assertIsInstance(sweep_val, tuple) + self.assertEqual(len(sweep_val), 2) + if sweep_val[0] == "s1": + val1 = f"s1-{sweep_val[1]}" + if val1: + values.remove(val1) + self.assertEqual(len(values), 0) + + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one", sweep={ + "s1":{ + SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1 + }, + "s2":{ + SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15 + } + }) + + es = pattern_one.expand_sweeps() + + self.assertIsInstance(es, list) + self.assertEqual(len(es), 15) + + values = [ + "s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20", + "s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35", + "s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50", + "s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65", + "s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80", + ] + + for sweep_vals in es: + self.assertIsInstance(sweep_vals, tuple) + self.assertEqual(len(sweep_vals), 2) + + val1 = None + val2 = None + for sweep_val in sweep_vals: + self.assertIsInstance(sweep_val, tuple) + self.assertEqual(len(sweep_val), 2) + if sweep_val[0] == "s1": + val1 = f"s1-{sweep_val[1]}" + if sweep_val[0] == "s2": + val2 = f"s2-{sweep_val[1]}" + if val1 and val2: + values.remove(f"{val1}/{val2}") + self.assertEqual(len(values), 0) + # Test that BaseRecipe instantiation def testBaseRule(self)->None: with self.assertRaises(TypeError): @@ -87,7 +160,7 @@ class MeowTests(unittest.TestCase): pass def _is_valid_pattern(self, pattern:Any)->None: pass - FullRule("name", "", "") + FullRule("name", valid_pattern_one, valid_recipe_one) # Test that create_rule creates a rule from pattern and recipe def testCreateRule(self)->None: @@ -227,5 +300,3 @@ class MeowTests(unittest.TestCase): pass FullTestConductor() - - # TODO Test expansion of parameter sweeps diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 8c78977..dac9af3 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -7,20 +7,25 @@ from multiprocessing import Pipe from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \ - JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \ - PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \ - PARAMS_FILE, JOB_FILE, RESULT_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START + JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \ + PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, JOB_ERROR, \ + PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \ + get_base_file, get_job_file, get_result_file from core.correctness.validation import valid_job from core.functionality import get_file_hash, create_job, \ - create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml + create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml, \ + write_file, lines_to_string from core.meow import create_rules, create_rule from patterns.file_event_pattern import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ - PapermillHandler, job_func -from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule -from shared import setup, teardown, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \ + PapermillHandler, papermill_job_func +from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func +from rules import FileEventJupyterNotebookRule, FileEventPythonRule +from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \ + COMPLETE_PYTHON_SCRIPT, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \ TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK + class JupyterNotebookTests(unittest.TestCase): def setUp(self)->None: super().setUp() @@ -349,7 +354,7 @@ class JupyterNotebookTests(unittest.TestCase): } job_dict = create_job( - JOB_TYPE_PYTHON, + JOB_TYPE_PAPERMILL, create_watchdog_event( file_path, rule, @@ -359,7 +364,7 @@ class JupyterNotebookTests(unittest.TestCase): extras={ JOB_PARAMETERS:params_dict, JOB_HASH: file_hash, - PYTHON_FUNC:job_func, + PYTHON_FUNC:papermill_job_func, PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT, PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE } @@ -375,25 +380,35 @@ class JupyterNotebookTests(unittest.TestCase): param_file = os.path.join(job_dir, PARAMS_FILE) write_yaml(params_dict, param_file) - base_file = os.path.join(job_dir, BASE_FILE) + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL)) write_notebook(APPENDING_NOTEBOOK, base_file) - job_func(job_dict) + papermill_job_func(job_dict) job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID]) self.assertTrue(os.path.exists(job_dir)) - self.assertTrue(os.path.exists(os.path.join(job_dir, META_FILE))) - self.assertTrue(os.path.exists(os.path.join(job_dir, BASE_FILE))) + + meta_path = os.path.join(job_dir, META_FILE) + self.assertTrue(os.path.exists(meta_path)) + status = read_yaml(meta_path) + self.assertIsInstance(status, dict) + self.assertIn(JOB_STATUS, status) + self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS]) + + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL)))) self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE))) - self.assertTrue(os.path.exists(os.path.join(job_dir, JOB_FILE))) - self.assertTrue(os.path.exists(os.path.join(job_dir, RESULT_FILE))) + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL)))) + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL)))) self.assertTrue(os.path.exists(result_path)) # Test jobFunc doesn't execute with no args def testJobFuncBadArgs(self)->None: try: - job_func({}) + papermill_job_func({}) except Exception: pass @@ -411,3 +426,363 @@ class PythonTests(unittest.TestCase): def tearDown(self)->None: super().tearDown() teardown() + + # Test PythonRecipe can be created + def testPythonRecipeCreationMinimum(self)->None: + PythonRecipe("test_recipe", BAREBONES_PYTHON_SCRIPT) + + # Test PythonRecipe cannot be created without name + def testPythonRecipeCreationNoName(self)->None: + with self.assertRaises(ValueError): + PythonRecipe("", BAREBONES_PYTHON_SCRIPT) + + # Test PythonRecipe cannot be created with invalid name + def testPythonRecipeCreationInvalidName(self)->None: + with self.assertRaises(ValueError): + PythonRecipe("@test_recipe", BAREBONES_PYTHON_SCRIPT) + + # Test PythonRecipe cannot be created with invalid recipe + def testPythonRecipeCreationInvalidRecipe(self)->None: + with self.assertRaises(TypeError): + PythonRecipe("test_recipe", BAREBONES_NOTEBOOK) + + # Test PythonRecipe name setup correctly + def testPythonRecipeSetupName(self)->None: + name = "name" + pr = PythonRecipe(name, BAREBONES_PYTHON_SCRIPT) + self.assertEqual(pr.name, name) + + # Test PythonRecipe recipe setup correctly + def testPythonRecipeSetupRecipe(self)->None: + pr = PythonRecipe("name", BAREBONES_PYTHON_SCRIPT) + self.assertEqual(pr.recipe, BAREBONES_PYTHON_SCRIPT) + + # Test PythonRecipe parameters setup correctly + def testPythonRecipeSetupParameters(self)->None: + parameters = { + "a": 1, + "b": True + } + pr = PythonRecipe( + "name", BAREBONES_PYTHON_SCRIPT, parameters=parameters) + self.assertEqual(pr.parameters, parameters) + + # Test PythonRecipe requirements setup correctly + def testPythonRecipeSetupRequirements(self)->None: + requirements = { + "a": 1, + "b": True + } + pr = PythonRecipe( + "name", BAREBONES_PYTHON_SCRIPT, requirements=requirements) + self.assertEqual(pr.requirements, requirements) + + # Test PythonHandler can be created + def testPythonHandlerMinimum(self)->None: + PythonHandler( + TEST_HANDLER_BASE, + TEST_JOB_OUTPUT + ) + + # Test PythonHandler will handle given events + def testPythonHandlerHandling(self)->None: + from_handler_reader, from_handler_writer = Pipe() + ph = PythonHandler( + TEST_HANDLER_BASE, + TEST_JOB_OUTPUT + ) + ph.to_runner = from_handler_writer + + with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f: + f.write("Data") + + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one") + recipe = PythonRecipe( + "recipe_one", COMPLETE_PYTHON_SCRIPT) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + rules = create_rules(patterns, recipes) + self.assertEqual(len(rules), 1) + _, rule = rules.popitem() + self.assertIsInstance(rule, FileEventPythonRule) + + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + event = { + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), + WATCHDOG_BASE: TEST_MONITOR_BASE, + EVENT_RULE: rule, + WATCHDOG_HASH: get_file_hash( + os.path.join(TEST_MONITOR_BASE, "A"), SHA256 + ) + } + + ph.handle(event) + + if from_handler_reader.poll(3): + job_dir = from_handler_reader.recv() + + self.assertIsInstance(job_dir, str) + self.assertTrue(os.path.exists(job_dir)) + + job = read_yaml(os.path.join(job_dir, META_FILE)) + valid_job(job) + + # Test PythonHandler will create enough jobs from single sweep + def testPythonHandlerHandlingSingleSweep(self)->None: + from_handler_reader, from_handler_writer = Pipe() + ph = PythonHandler( + TEST_HANDLER_BASE, + TEST_JOB_OUTPUT + ) + ph.to_runner = from_handler_writer + + with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f: + f.write("Data") + + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one", sweep={"s":{ + SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1 + }}) + recipe = PythonRecipe( + "recipe_one", COMPLETE_PYTHON_SCRIPT) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + rules = create_rules(patterns, recipes) + self.assertEqual(len(rules), 1) + _, rule = rules.popitem() + self.assertIsInstance(rule, FileEventPythonRule) + + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + event = { + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), + WATCHDOG_BASE: TEST_MONITOR_BASE, + EVENT_RULE: rule, + WATCHDOG_HASH: get_file_hash( + os.path.join(TEST_MONITOR_BASE, "A"), SHA256 + ) + } + + ph.handle(event) + + jobs = [] + recieving = True + while recieving: + if from_handler_reader.poll(3): + jobs.append(from_handler_reader.recv()) + else: + recieving = False + + values = [0, 1, 2] + self.assertEqual(len(jobs), 3) + for job_dir in jobs: + self.assertIsInstance(job_dir, str) + self.assertTrue(os.path.exists(job_dir)) + + job = read_yaml(os.path.join(job_dir, META_FILE)) + valid_job(job) + + self.assertIn(JOB_PARAMETERS, job) + self.assertIn("s", job[JOB_PARAMETERS]) + if job[JOB_PARAMETERS]["s"] in values: + values.remove(job[JOB_PARAMETERS]["s"]) + self.assertEqual(len(values), 0) + + # Test PythonHandler will create enough jobs from multiple sweeps + def testPythonHandlerHandlingMultipleSweep(self)->None: + from_handler_reader, from_handler_writer = Pipe() + ph = PythonHandler( + TEST_HANDLER_BASE, + TEST_JOB_OUTPUT + ) + ph.to_runner = from_handler_writer + + with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f: + f.write("Data") + + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one", sweep={ + "s1":{ + SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1 + }, + "s2":{ + SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15 + } + }) + recipe = PythonRecipe( + "recipe_one", COMPLETE_PYTHON_SCRIPT) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + rules = create_rules(patterns, recipes) + self.assertEqual(len(rules), 1) + _, rule = rules.popitem() + self.assertIsInstance(rule, FileEventPythonRule) + + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + event = { + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), + WATCHDOG_BASE: TEST_MONITOR_BASE, + EVENT_RULE: rule, + WATCHDOG_HASH: get_file_hash( + os.path.join(TEST_MONITOR_BASE, "A"), SHA256 + ) + } + + ph.handle(event) + + jobs = [] + recieving = True + while recieving: + if from_handler_reader.poll(3): + jobs.append(from_handler_reader.recv()) + else: + recieving = False + + values = [ + "s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20", + "s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35", + "s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50", + "s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65", + "s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80", + ] + self.assertEqual(len(jobs), 15) + for job_dir in jobs: + self.assertIsInstance(job_dir, str) + self.assertTrue(os.path.exists(job_dir)) + + job = read_yaml(os.path.join(job_dir, META_FILE)) + valid_job(job) + + self.assertIn(JOB_PARAMETERS, job) + val1 = None + val2 = None + if "s1" in job[JOB_PARAMETERS]: + val1 = f"s1-{job[JOB_PARAMETERS]['s1']}" + if "s2" in job[JOB_PARAMETERS]: + val2 = f"s2-{job[JOB_PARAMETERS]['s2']}" + val = None + if val1 and val2: + val = f"{val1}/{val2}" + if val and val in values: + values.remove(val) + self.assertEqual(len(values), 0) + + # Test jobFunc performs as expected + def testJobFunc(self)->None: + file_path = os.path.join(TEST_MONITOR_BASE, "test") + result_path = os.path.join(TEST_MONITOR_BASE, "output") + + with open(file_path, "w") as f: + f.write("250") + + file_hash = get_file_hash(file_path, SHA256) + + pattern = FileEventPattern( + "pattern", + file_path, + "recipe_one", + "infile", + parameters={ + "extra":"A line from a test Pattern", + "outfile": result_path + }) + recipe = PythonRecipe( + "recipe_one", COMPLETE_PYTHON_SCRIPT) + + rule = create_rule(pattern, recipe) + + params_dict = { + "extra":"extra", + "infile":file_path, + "outfile": result_path + } + + job_dict = create_job( + JOB_TYPE_PYTHON, + create_watchdog_event( + file_path, + rule, + TEST_MONITOR_BASE, + file_hash + ), + extras={ + JOB_PARAMETERS:params_dict, + JOB_HASH: file_hash, + PYTHON_FUNC:python_job_func, + PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT, + PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE + } + ) + + job_dir = os.path.join( + job_dict[PYTHON_EXECUTION_BASE], job_dict[JOB_ID]) + make_dir(job_dir) + + meta_file = os.path.join(job_dir, META_FILE) + write_yaml(job_dict, meta_file) + + param_file = os.path.join(job_dir, PARAMS_FILE) + write_yaml(params_dict, param_file) + + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON)) + write_notebook(APPENDING_NOTEBOOK, base_file) + write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file) + + python_job_func(job_dict) + + job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID]) + self.assertTrue(os.path.exists(job_dir)) + meta_path = os.path.join(job_dir, META_FILE) + self.assertTrue(os.path.exists(meta_path)) + + status = read_yaml(meta_path) + self.assertIsInstance(status, dict) + self.assertIn(JOB_STATUS, status) + self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS]) + self.assertNotIn(JOB_ERROR, status) + + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON)))) + self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE))) + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON)))) + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON)))) + + self.assertTrue(os.path.exists(result_path)) + + # Test jobFunc doesn't execute with no args + def testJobFuncBadArgs(self)->None: + try: + python_job_func({}) + except Exception: + pass + + self.assertEqual(len(os.listdir(TEST_HANDLER_BASE)), 0) + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + # TODO test default parameter function execution \ No newline at end of file diff --git a/tests/test_runner.py b/tests/test_runner.py index 57b6620..dda3a07 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -6,7 +6,7 @@ import unittest from time import sleep from conductors import LocalPythonConductor -from core.correctness.vars import RESULT_FILE +from core.correctness.vars import get_result_file, JOB_TYPE_PAPERMILL from core.functionality import make_dir, read_notebook from core.meow import BaseMonitor, BaseHandler, BaseConductor from core.runner import MeowRunner @@ -187,7 +187,8 @@ class MeowTests(unittest.TestCase): job_dir = os.path.join(TEST_JOB_OUTPUT, job_id) self.assertEqual(len(os.listdir(job_dir)), 5) - result = read_notebook(os.path.join(job_dir, RESULT_FILE)) + result = read_notebook( + os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL))) self.assertIsNotNone(result) output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt") @@ -279,7 +280,8 @@ class MeowTests(unittest.TestCase): mid_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id) self.assertEqual(len(os.listdir(mid_job_dir)), 5) - result = read_notebook(os.path.join(mid_job_dir, RESULT_FILE)) + result = read_notebook( + os.path.join(mid_job_dir, get_result_file(JOB_TYPE_PAPERMILL))) self.assertIsNotNone(result) mid_output_path = os.path.join(TEST_MONITOR_BASE, "middle", "A.txt") @@ -293,7 +295,8 @@ class MeowTests(unittest.TestCase): final_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id) self.assertEqual(len(os.listdir(final_job_dir)), 5) - result = read_notebook(os.path.join(final_job_dir, RESULT_FILE)) + result = read_notebook(os.path.join(final_job_dir, + get_result_file(JOB_TYPE_PAPERMILL))) self.assertIsNotNone(result) final_output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt") @@ -305,8 +308,8 @@ class MeowTests(unittest.TestCase): self.assertEqual(data, "Initial Data\nA line from Pattern 1\nA line from Pattern 2") - # TODO sweep tests - # TODO adding tests with numpy + # TODO sweep execution test + # TODO adding tests with numpy or other external dependency # TODO test getting job cannot handle # TODO test getting event cannot handle # TODO test with several matched monitors diff --git a/tests/test_validation.py b/tests/test_validation.py index 8a0f169..6829f2b 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -9,7 +9,7 @@ from typing import Any, Union from core.correctness.validation import check_type, check_implementation, \ valid_string, valid_dict, valid_list, valid_existing_file_path, \ valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \ - setup_debugging, valid_watchdog_event + setup_debugging, valid_watchdog_event, check_callable from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \ EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \ JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \ @@ -304,7 +304,7 @@ class CorrectnessTests(unittest.TestCase): with self.assertRaises(TypeError): setup_debugging(stream, "1") - #Test watchdog event dict + # Test watchdog event dict def testWatchdogEventValidation(self)->None: valid_watchdog_event({ EVENT_TYPE: "test", @@ -336,4 +336,11 @@ class CorrectnessTests(unittest.TestCase): valid_event({"EVENT_TYPE": "test"}) with self.assertRaises(KeyError): - valid_event({}) \ No newline at end of file + valid_event({}) + + # Test check_callable + def testCheckCallable(self)->None: + check_callable(make_dir) + + with self.assertRaises(TypeError): + check_callable("a")