differentiated papermill and python jobs more clearly

conductors/local_python_conductor.py

@@ -14,7 +14,7 @@ from typing import Any, Tuple
 from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
     STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
     STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, PYTHON_OUTPUT_DIR, \
-    JOB_TYPE
+    JOB_TYPE, JOB_TYPE_PAPERMILL
 from core.correctness.validation import valid_job
 from core.functionality import read_yaml, write_yaml
 from core.meow import BaseConductor
@@ -30,7 +30,7 @@ class LocalPythonConductor(BaseConductor):
         process it or not. This conductor will accept any Python job type"""
         try:
             valid_job(job)
-            if job[JOB_TYPE] == JOB_TYPE_PYTHON:
+            if job[JOB_TYPE] in [JOB_TYPE_PYTHON, JOB_TYPE_PAPERMILL]:
                 return True, ""
         except Exception as e:
             pass

core/correctness/validation.py

@@ -72,6 +72,12 @@ def check_type(variable:Any, expected_type:type, alt_types:list[type]=[],
             % (get_args(expected_type), type(variable))
         )
 
+def check_callable(call:Any)->None:
+    """Checks if a given variable is a callable function. Raises TypeError if 
+    not."""
+    if not callable(call):
+        raise TypeError(f"Given object '{call}' is not a callable function")
+
 def check_implementation(child_func, parent_class):
     """Checks if the given function has been overridden from the one inherited
     from the parent class. Raises a NotImplementedError if this is the case."""
@@ -94,6 +100,14 @@ def check_implementation(child_func, parent_class):
         msg = get_not_imp_msg(parent_class, parent_func)
         raise NotImplementedError(msg)
 
+def check_script(script:Any):
+    """Checks if a given variable is a valid script. Raises TypeError if 
+    not."""
+    # TODO investigate more robust check here
+    check_type(script, list)
+    for line in script:
+        check_type(line, str)
+
 def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
     """Checks that all characters in a given string are present in a provided
     list of characters. Will raise an ValueError if unexpected character is

core/correctness/vars.py

@@ -80,10 +80,24 @@ DIR_EVENTS = [
 # meow jobs
 JOB_TYPE = "job_type"
 JOB_TYPE_PYTHON = "python"
+JOB_TYPE_PAPERMILL = "papermill"
 PYTHON_FUNC = "func"
 PYTHON_EXECUTION_BASE = "exection_base"
 PYTHON_OUTPUT_DIR = "output_dir"
 
+JOB_TYPES = {
+    JOB_TYPE_PAPERMILL: [
+        "base.ipynb",
+        "job.ipynb",
+        "result.ipynb",
+    ],
+    JOB_TYPE_PYTHON: [
+        "base.py",
+        "job.py",
+        "result.py",
+    ]
+}
+
 # job definitions
 JOB_ID = "id"
 JOB_EVENT = "event"
@@ -108,10 +122,7 @@ STATUS_DONE = "done"
 
 # job definition files
 META_FILE = "job.yml"
-BASE_FILE = "base.ipynb"
 PARAMS_FILE = "params.yml"
-JOB_FILE = "job.ipynb"
-RESULT_FILE = "result.ipynb"
 
 # Parameter sweep keys
 SWEEP_START = "start"
@@ -132,3 +143,12 @@ def get_not_imp_msg(parent_class, class_function):
     return f"Children of the '{parent_class.__name__}' class must implement " \
         f"the '{class_function.__name__}({signature(class_function)})' " \
         "function"
+
+def get_base_file(job_type:str):
+    return JOB_TYPES[job_type][0]
+
+def get_job_file(job_type:str):
+    return JOB_TYPES[job_type][1]
+
+def get_result_file(job_type:str):
+    return JOB_TYPES[job_type][2]

core/functionality.py

@@ -1,4 +1,4 @@
-
+# TODO comments
 import copy
 import hashlib
 import json
@@ -15,7 +15,7 @@ from typing import Any
 from random import SystemRandom
 
 from core.correctness.validation import check_type, valid_existing_file_path, \
-    valid_path
+    valid_path, check_script
 from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
     VALID_CHANNELS, HASH_BUFFER_SIZE, SHA256, DEBUG_WARNING, DEBUG_INFO, \
     EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
@@ -128,6 +128,18 @@ def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):
                 
     os.makedirs(path, exist_ok=can_exist)
     
+def read_file(filepath:str):
+    with open(filepath, 'r') as file:
+        return file.read()
+
+def read_file_lines(filepath:str):
+    with open(filepath, 'r') as file:
+        return file.readlines()
+
+def write_file(source:str, filename:str):
+    with open(filename, 'w') as file:
+        file.write(source)
+
 def read_yaml(filepath:str):
     """
     Reads a file path as a yaml object.
@@ -171,7 +183,7 @@ def write_notebook(source:dict[str,Any], filename:str):
         json.dump(source, job_file)
 
 # Adapted from: https://github.com/rasmunk/notebook_parameterizer
-def parameterize_jupyter_notebook( jupyter_notebook:dict[str,Any], 
+def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any], 
         parameters:dict[str,Any], expand_env_values:bool=False)->dict[str,Any]:
     nbformat.validate(jupyter_notebook)
     check_type(parameters, dict)
@@ -244,6 +256,38 @@ def parameterize_jupyter_notebook( jupyter_notebook:dict[str,Any],
 
     return output_notebook
 
+def parameterize_python_script(script:list[str], parameters:dict[str,Any], 
+        expand_env_values:bool=False)->dict[str,Any]:
+    check_script(script)
+    check_type(parameters, dict)
+
+    output_script = copy.deepcopy(script)
+
+    for i, line in enumerate(output_script):
+        if "=" in line:
+            d_line = list(map(lambda x: x.replace(" ", ""), 
+                line.split("=")))
+            # Matching parameter name
+            if len(d_line) == 2 and d_line[0] in parameters:
+                value = parameters[d_line[0]]
+                # Whether to expand value from os env
+                if (
+                    expand_env_values
+                    and isinstance(value, str)
+                    and value.startswith("ENV_")
+                ):
+                    env_var = value.replace("ENV_", "")
+                    value = os.getenv(
+                        env_var, 
+                        "MISSING ENVIRONMENT VARIABLE: {}".format(env_var)
+                    )
+                output_script[i] = f"{d_line[0]} = {repr(value)}"
+                
+    # Validate that the parameterized notebook is still valid
+    check_script(output_script)
+
+    return output_script
+
 def print_debug(print_target, debug_level, msg, level)->None:
         if print_target is None:
             return
@@ -338,3 +382,6 @@ def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
     }
 
     return {**extras, **job_dict}
+
+def lines_to_string(lines:list[str])->str:
+    return "\n".join(lines)

core/meow.py

@@ -98,7 +98,6 @@ class BasePattern:
         check_implementation(type(self)._is_valid_recipe, BasePattern)
         check_implementation(type(self)._is_valid_parameters, BasePattern)
         check_implementation(type(self)._is_valid_output, BasePattern)
-        check_implementation(type(self)._is_valid_sweep, BasePattern)
         self._is_valid_name(name)
         self.name = name
         self._is_valid_recipe(recipe)
@@ -140,8 +139,9 @@ class BasePattern:
         pass
 
     def _is_valid_sweep(self, sweep:dict[str,Union[int,float,complex]])->None:
-        """Validation check for 'sweep' variable from main constructor. Must 
+        """Validation check for 'sweep' variable from main constructor. This 
-        be implemented by any child class."""
+        function is implemented to check for the types given in the signature, 
+        and must be overridden if these differ."""
         check_type(sweep, dict)
         if not sweep:
             return
@@ -208,13 +208,19 @@ class BaseRule:
         the input parameters."""
         check_implementation(type(self)._is_valid_pattern, BaseRule)
         check_implementation(type(self)._is_valid_recipe, BaseRule)
+        self.__check_types_set()
         self._is_valid_name(name)
         self.name = name
         self._is_valid_pattern(pattern)
         self.pattern = pattern
         self._is_valid_recipe(recipe)
         self.recipe = recipe
-        self.__check_types_set()
+        check_type(pattern, BasePattern)
+        check_type(recipe, BaseRecipe)
+        if pattern.recipe != recipe.name:
+            raise ValueError(f"Cannot create Rule {name}. Pattern "
+                f"{pattern.name} does not identify Recipe {recipe.name}. It "
+                f"uses {pattern.recipe}")
 
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 

patterns/file_event_pattern.py

@@ -23,8 +23,8 @@ from core.correctness.validation import check_type, valid_string, \
     setup_debugging
 from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
     VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
-    FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
+    FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
-    WATCHDOG_BASE, FILE_RETROACTIVE_EVENT, WATCHDOG_HASH, SHA256
+    FILE_RETROACTIVE_EVENT, SHA256
 from core.functionality import print_debug, create_watchdog_event, \
     get_file_hash, create_fake_watchdog_event
 from core.meow import BasePattern, BaseMonitor, BaseRule, BaseRecipe, \

recipes/jupyter_notebook_recipe.py

@@ -6,7 +6,6 @@ notebooks, along with an appropriate handler for said events.
 Author(s): David Marchant
 """
 import os
-import itertools
 import nbformat
 import sys
 
@@ -17,12 +16,12 @@ from core.correctness.validation import check_type, valid_string, \
     valid_event
 from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \
-    EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
+    EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \
-    PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \
+    PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, \
     PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
-    EVENT_RULE
+    EVENT_RULE, get_base_file, get_job_file, get_result_file
 from core.functionality import print_debug, create_job, replace_keywords, \
-    make_dir, write_yaml, write_notebook
+    make_dir, write_yaml, write_notebook, read_notebook
 from core.meow import BaseRecipe, BaseHandler
 
 
@@ -140,19 +139,19 @@ class PapermillHandler(BaseHandler):
         """Function to set up new job dict and send it to the runner to be 
         executed."""
         meow_job = create_job(
-            JOB_TYPE_PYTHON, 
+            JOB_TYPE_PAPERMILL, 
             event, 
             extras={
                 JOB_PARAMETERS:yaml_dict,
                 JOB_HASH: event[WATCHDOG_HASH],
-                PYTHON_FUNC:job_func,
+                PYTHON_FUNC:papermill_job_func,
                 PYTHON_OUTPUT_DIR:self.output_dir,
                 PYTHON_EXECUTION_BASE:self.handler_base
             }
         )
         print_debug(self._print_target, self.debug_level,  
             f"Creating job from event at {event[EVENT_PATH]} of type "
-            f"{JOB_TYPE_PYTHON}.", DEBUG_INFO)
+            f"{JOB_TYPE_PAPERMILL}.", DEBUG_INFO)
 
         # replace MEOW keyworks within variables dict
         yaml_dict = replace_keywords(
@@ -172,7 +171,7 @@ class PapermillHandler(BaseHandler):
         write_yaml(meow_job, meta_file)
 
         # write an executable notebook to the job directory
-        base_file = os.path.join(job_dir, BASE_FILE)
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
         write_notebook(event[EVENT_RULE].recipe.recipe, base_file)
 
         # write a parameter file to the job directory
@@ -188,25 +187,26 @@ class PapermillHandler(BaseHandler):
         self.to_runner.send(job_dir)
 
 # Papermill job execution code, to be run within the conductor
-def job_func(job):
+def papermill_job_func(job):
     # Requires own imports as will be run in its own execution environment
     import os
     import papermill
     from datetime import datetime
     from core.functionality import write_yaml, read_yaml, write_notebook, \
         get_file_hash, parameterize_jupyter_notebook
-    from core.correctness.vars import JOB_EVENT, EVENT_RULE, JOB_ID, \
+    from core.correctness.vars import JOB_EVENT, JOB_ID, \
-        EVENT_PATH, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE, \
+        EVENT_PATH, META_FILE, PARAMS_FILE, \
         JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
-        JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE
+        JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE, get_job_file, \
-
+        get_result_file
-    event = job[JOB_EVENT]
 
     # Identify job files
     job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID])
     meta_file = os.path.join(job_dir, META_FILE)
-    job_file = os.path.join(job_dir, JOB_FILE)
+    # TODO fix these paths so they are dynamic
-    result_file = os.path.join(job_dir, RESULT_FILE)
+    base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
+    job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL))
+    result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL))
     param_file = os.path.join(job_dir, PARAMS_FILE)
 
     yaml_dict = read_yaml(param_file)
@@ -233,8 +233,10 @@ def job_func(job):
 
     # Create a parameterised version of the executable notebook
     try:
+        base_notebook = read_notebook(base_file)
+        # TODO read notebook from already written file rather than event
         job_notebook = parameterize_jupyter_notebook(
-            event[EVENT_RULE].recipe.recipe, yaml_dict
+            base_notebook, yaml_dict
         )
         write_notebook(job_notebook, job_file)
     except Exception as e:

recipes/python_recipe.py

@@ -6,36 +6,34 @@ along with an appropriate handler for said events.
 Author(s): David Marchant
 """
 import os
-import itertools
-import nbformat
 import sys
 
 from typing import Any, Tuple
 
-from core.correctness.validation import check_type, valid_string, \
+from core.correctness.validation import check_script, valid_string, \
     valid_dict, valid_event, valid_existing_dir_path, setup_debugging
 from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, PYTHON_EXECUTION_BASE, \
     EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
-    PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, BASE_FILE, \
+    PYTHON_OUTPUT_DIR, JOB_ID, WATCHDOG_BASE, META_FILE, \
-    PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE
+    PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
+    get_job_file, get_base_file, get_result_file
 from core.functionality import print_debug, create_job, replace_keywords, \
-    make_dir, write_yaml, write_notebook
+    make_dir, write_yaml, write_file, lines_to_string, read_file_lines
 from core.meow import BaseRecipe, BaseHandler
 
 
 class PythonRecipe(BaseRecipe):
-    def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={}, 
+    def __init__(self, name:str, recipe:list[str], parameters:dict[str,Any]={}, 
             requirements:dict[str,Any]={}):
         """PythonRecipe Constructor. This is used to execute python analysis 
         code."""
         super().__init__(name, recipe, parameters, requirements)
 
-    def _is_valid_recipe(self, recipe:dict[str,Any])->None:
+    def _is_valid_recipe(self, recipe:list[str])->None:
         """Validation check for 'recipe' variable from main constructor. 
         Called within parent BaseRecipe constructor."""
-        check_type(recipe, dict)
+        check_script(recipe)
-        nbformat.validate(recipe)
 
     def _is_valid_parameters(self, parameters:dict[str,Any])->None:
         """Validation check for 'parameters' variable from main constructor. 
@@ -135,7 +133,7 @@ class PythonHandler(BaseHandler):
             extras={
                 JOB_PARAMETERS:yaml_dict,
                 JOB_HASH: event[WATCHDOG_HASH],
-                PYTHON_FUNC:job_func,
+                PYTHON_FUNC:python_job_func,
                 PYTHON_OUTPUT_DIR:self.output_dir,
                 PYTHON_EXECUTION_BASE:self.handler_base
             }
@@ -161,9 +159,9 @@ class PythonHandler(BaseHandler):
         meta_file = os.path.join(job_dir, META_FILE)
         write_yaml(meow_job, meta_file)
 
-        # write an executable notebook to the job directory
+        # write an executable script to the job directory
-        base_file = os.path.join(job_dir, BASE_FILE)
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
-        write_notebook(event[EVENT_RULE].recipe.recipe, base_file)
+        write_file(lines_to_string(event[EVENT_RULE].recipe.recipe), base_file)
 
         # write a parameter file to the job directory
         param_file = os.path.join(job_dir, PARAMS_FILE)
@@ -178,25 +176,26 @@ class PythonHandler(BaseHandler):
         self.to_runner.send(job_dir)
 
 # Papermill job execution code, to be run within the conductor
-def job_func(job):
+def python_job_func(job):
     # Requires own imports as will be run in its own execution environment
+    import sys
     import os
-    import papermill
     from datetime import datetime
-    from core.functionality import write_yaml, read_yaml, write_notebook, \
+    from io import StringIO
-        get_file_hash, parameterize_jupyter_notebook
+    from core.functionality import write_yaml, read_yaml, \
-    from core.correctness.vars import JOB_EVENT, EVENT_RULE, JOB_ID, \
+        get_file_hash, parameterize_python_script
-        EVENT_PATH, META_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE, \
+    from core.correctness.vars import JOB_EVENT, JOB_ID, \
+        EVENT_PATH, META_FILE, PARAMS_FILE, \
         JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
-        JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE
+        JOB_ERROR, STATUS_FAILED, PYTHON_EXECUTION_BASE, get_base_file, \
-
+        get_job_file, get_result_file
-    event = job[JOB_EVENT]
 
     # Identify job files
     job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID])
     meta_file = os.path.join(job_dir, META_FILE)
-    job_file = os.path.join(job_dir, JOB_FILE)
+    base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
-    result_file = os.path.join(job_dir, RESULT_FILE)
+    job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON))
+    result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON))
     param_file = os.path.join(job_dir, PARAMS_FILE)
 
     yaml_dict = read_yaml(param_file)
@@ -221,12 +220,13 @@ def job_func(job):
             write_yaml(job, meta_file)
             return
 
-    # Create a parameterised version of the executable notebook
+    # Create a parameterised version of the executable script
     try:
-        job_notebook = parameterize_jupyter_notebook(
+        base_script = read_file_lines(base_file)
-            event[EVENT_RULE].recipe.recipe, yaml_dict
+        job_script = parameterize_python_script(
+            base_script, yaml_dict
         )
-        write_notebook(job_notebook, job_file)
+        write_file(lines_to_string(job_script), job_file)
     except Exception as e:
         job[JOB_STATUS] = STATUS_FAILED
         job[JOB_END_TIME] = datetime.now()
@@ -235,13 +235,33 @@ def job_func(job):
         write_yaml(job, meta_file)
         return
 
-    # Execute the parameterised notebook
+    # Execute the parameterised script
+    std_stdout = sys.stdout
+    std_stderr = sys.stderr
     try:
-        papermill.execute_notebook(job_file, result_file, {})
+        redirected_output = sys.stdout
+        redirected_error = sys.stderr
+
+        exec(open(job_file).read())        
+
+        write_file(f"""--STDOUT--
+            {redirected_output}
+            
+            --STDERR--
+            {redirected_error}
+            """, 
+            result_file)
+
     except Exception as e:
+        sys.stdout = std_stdout
+        sys.stderr = std_stderr
+
         job[JOB_STATUS] = STATUS_FAILED
         job[JOB_END_TIME] = datetime.now()
         msg = f"Result file {result_file} was not created successfully. {e}"
         job[JOB_ERROR] = msg
         write_yaml(job, meta_file)
         return
+
+    sys.stdout = std_stdout
+    sys.stderr = std_stderr

rules/__init__.py

@@ -1,2 +1,3 @@
 
 from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
+from rules.file_event_python_rule import FileEventPythonRule

rules/file_event_python_rule.py

@@ -0,0 +1,30 @@
+
+"""
+This file contains definitions for a MEOW rule connecting the FileEventPattern
+and PythonRecipe.
+
+Author(s): David Marchant
+"""
+from core.correctness.validation import check_type
+from core.meow import BaseRule
+from patterns.file_event_pattern import FileEventPattern
+from recipes.python_recipe import PythonRecipe
+
+# TODO potentailly remove this and just invoke BaseRule directly, as does not
+# add any functionality other than some validation.
+class FileEventPythonRule(BaseRule):
+    pattern_type = "FileEventPattern"
+    recipe_type = "PythonRecipe" 
+    def __init__(self, name: str, pattern:FileEventPattern, 
+            recipe:PythonRecipe):
+        super().__init__(name, pattern, recipe)
+
+    def _is_valid_pattern(self, pattern:FileEventPattern)->None:
+        """Validation check for 'pattern' variable from main constructor. Is 
+        automatically called during initialisation."""
+        check_type(pattern, FileEventPattern)
+
+    def _is_valid_recipe(self, recipe:PythonRecipe)->None:
+        """Validation check for 'recipe' variable from main constructor. Is 
+        automatically called during initialisation."""
+        check_type(recipe, PythonRecipe)

tests/shared.py

@@ -22,6 +22,31 @@ def teardown():
     rmtree(TEST_JOB_OUTPUT)
     rmtree("first")
 
+# Recipe funcs
+BAREBONES_PYTHON_SCRIPT = [
+    ""
+]
+COMPLETE_PYTHON_SCRIPT = [
+    "# Setup parameters",
+    "num = 1000",
+    "infile = 'somehere/particular'",
+    "outfile = 'nowhere/particular'",
+    "",
+    "with open(infile, 'r') as file:",
+    "    s = int(file.read())",
+    ""
+    "for i in range(num):",
+    "    s += i",
+    "",
+    "div_by = 4",
+    "result = s / div_by",
+    "",
+    "print(result)",
+    "",
+    "with open(outfile, 'w') as file:",
+    "    file.write(str(result))"
+]
+
 # Jupyter notebooks
 BAREBONES_NOTEBOOK = {
     "cells": [],

tests/test_conductors.py

@@ -4,15 +4,20 @@ import unittest
 
 from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
     JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, \
-    META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
+    META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, \
+    STATUS_DONE, JOB_TYPE_PAPERMILL, get_base_file, get_result_file, \
+    get_job_file
 from core.functionality import get_file_hash, create_watchdog_event, \
-    create_job, make_dir, write_yaml, write_notebook
+    create_job, make_dir, write_yaml, write_notebook, read_yaml, write_file, \
+    lines_to_string
 from core.meow import create_rule
 from conductors import LocalPythonConductor
 from patterns import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func
+from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
+    papermill_job_func
+from recipes.python_recipe import PythonRecipe, python_job_func
 from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
-    TEST_JOB_OUTPUT, TEST_HANDLER_BASE
+    TEST_JOB_OUTPUT, TEST_HANDLER_BASE, COMPLETE_PYTHON_SCRIPT
 
 
 def failing_func():
@@ -32,10 +37,94 @@ class MeowTests(unittest.TestCase):
     def testLocalPythonConductorCreation(self)->None:
         LocalPythonConductor()
 
-    #TODO Test LocalPythonConductor execution criteria
+    #TODO Test LocalPythonConductor executes valid python jobs
+    def testLocalPythonConductorValidPythonJob(self)->None:
+        lpc = LocalPythonConductor()
 
-    # Test LocalPythonConductor executes valid jobs
+        file_path = os.path.join(TEST_MONITOR_BASE, "test")
-    def testLocalPythonConductorValidJob(self)->None:
+        result_path = os.path.join(TEST_MONITOR_BASE, "output")
+
+        with open(file_path, "w") as f:
+            f.write("150")
+
+        file_hash = get_file_hash(file_path, SHA256)
+
+        pattern = FileEventPattern(
+            "pattern", 
+            file_path, 
+            "recipe_one", 
+            "infile", 
+            parameters={
+                "num":450,
+                "outfile":result_path
+            })
+        recipe = PythonRecipe(
+            "recipe_one", COMPLETE_PYTHON_SCRIPT)
+
+        rule = create_rule(pattern, recipe)
+
+        params_dict = {
+            "num":450,
+            "infile":file_path,
+            "outfile":result_path
+        }
+
+        job_dict = create_job(
+            JOB_TYPE_PYTHON,
+            create_watchdog_event(
+                file_path,
+                rule,
+                TEST_MONITOR_BASE,
+                file_hash
+            ),
+            extras={
+                JOB_PARAMETERS:params_dict,
+                JOB_HASH: file_hash,
+                PYTHON_FUNC:python_job_func,
+                PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
+                PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
+            }
+        )
+
+        job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
+        make_dir(job_dir)
+
+        param_file = os.path.join(job_dir, PARAMS_FILE)
+        write_yaml(params_dict, param_file)
+
+        meta_path = os.path.join(job_dir, META_FILE)
+        write_yaml(job_dict, meta_path)
+
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
+        write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
+
+        lpc.execute(job_dict)
+
+        self.assertFalse(os.path.exists(job_dir))
+        
+        output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
+        self.assertTrue(os.path.exists(output_dir))
+
+        meta_path = os.path.join(output_dir, META_FILE)
+        self.assertTrue(os.path.exists(meta_path))
+        status = read_yaml(meta_path)
+        self.assertIsInstance(status, dict)
+        self.assertIn(JOB_STATUS, status)
+        self.assertEqual(status[JOB_STATUS], STATUS_DONE)
+
+        self.assertNotIn(JOB_ERROR, status)
+        self.assertTrue(os.path.exists(
+            os.path.join(output_dir, get_base_file(JOB_TYPE_PYTHON))))
+        self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
+        self.assertTrue(os.path.exists(
+            os.path.join(output_dir, get_job_file(JOB_TYPE_PYTHON))))
+        self.assertTrue(os.path.exists(
+            os.path.join(output_dir, get_result_file(JOB_TYPE_PYTHON))))
+
+        self.assertTrue(os.path.exists(result_path))
+
+     # Test LocalPythonConductor executes valid papermill jobs
+    def testLocalPythonConductorValidPapermillJob(self)->None:
         lpc = LocalPythonConductor()
 
         file_path = os.path.join(TEST_MONITOR_BASE, "test")
@@ -67,7 +156,7 @@ class MeowTests(unittest.TestCase):
         }
 
         job_dict = create_job(
-            JOB_TYPE_PYTHON,
+            JOB_TYPE_PAPERMILL,
             create_watchdog_event(
                 file_path,
                 rule,
@@ -77,7 +166,7 @@ class MeowTests(unittest.TestCase):
             extras={
                 JOB_PARAMETERS:params_dict,
                 JOB_HASH: file_hash,
-                PYTHON_FUNC:job_func,
+                PYTHON_FUNC:papermill_job_func,
                 PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
                 PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
             }
@@ -89,7 +178,10 @@ class MeowTests(unittest.TestCase):
         param_file = os.path.join(job_dir, PARAMS_FILE)
         write_yaml(params_dict, param_file)
 
-        base_file = os.path.join(job_dir, BASE_FILE)
+        meta_path = os.path.join(job_dir, META_FILE)
+        write_yaml(job_dict, meta_path)
+
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
         write_notebook(APPENDING_NOTEBOOK, base_file)
 
         lpc.execute(job_dict)
@@ -99,11 +191,22 @@ class MeowTests(unittest.TestCase):
         
         output_dir = os.path.join(TEST_JOB_OUTPUT, job_dict[JOB_ID])
         self.assertTrue(os.path.exists(output_dir))
-        self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
+
-        self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
+
+        meta_path = os.path.join(output_dir, META_FILE)
+        self.assertTrue(os.path.exists(meta_path))
+        status = read_yaml(meta_path)
+        self.assertIsInstance(status, dict)
+        self.assertIn(JOB_STATUS, status)
+        self.assertEqual(status[JOB_STATUS], STATUS_DONE)
+
+        self.assertTrue(os.path.exists(
+            os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
         self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
-        self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
+        self.assertTrue(os.path.exists(
-        self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
+            os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
+        self.assertTrue(os.path.exists(
+            os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
 
         self.assertTrue(os.path.exists(result_path))
 
@@ -140,7 +243,7 @@ class MeowTests(unittest.TestCase):
         }
 
         bad_job_dict = create_job(
-            JOB_TYPE_PYTHON,
+            JOB_TYPE_PAPERMILL,
             create_watchdog_event(
                 file_path,
                 rule,
@@ -150,7 +253,7 @@ class MeowTests(unittest.TestCase):
             extras={
                 JOB_PARAMETERS:params_dict,
                 JOB_HASH: file_hash,
-                PYTHON_FUNC:job_func,
+                PYTHON_FUNC:papermill_job_func,
             }
         )
 
@@ -160,7 +263,7 @@ class MeowTests(unittest.TestCase):
         param_file = os.path.join(job_dir, PARAMS_FILE)
         write_yaml(params_dict, param_file)
 
-        base_file = os.path.join(job_dir, BASE_FILE)
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
         write_notebook(APPENDING_NOTEBOOK, base_file)
 
         with self.assertRaises(KeyError):
@@ -168,7 +271,7 @@ class MeowTests(unittest.TestCase):
 
         # Ensure execution can continue after one failed job
         good_job_dict = create_job(
-            JOB_TYPE_PYTHON,
+            JOB_TYPE_PAPERMILL,
             create_watchdog_event(
                 file_path,
                 rule,
@@ -178,7 +281,7 @@ class MeowTests(unittest.TestCase):
             extras={
                 JOB_PARAMETERS:params_dict,
                 JOB_HASH: file_hash,
-                PYTHON_FUNC:job_func,
+                PYTHON_FUNC:papermill_job_func,
                 PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
                 PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
             }
@@ -190,7 +293,7 @@ class MeowTests(unittest.TestCase):
         param_file = os.path.join(job_dir, PARAMS_FILE)
         write_yaml(params_dict, param_file)
 
-        base_file = os.path.join(job_dir, BASE_FILE)
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
         write_notebook(APPENDING_NOTEBOOK, base_file)
 
         lpc.execute(good_job_dict)
@@ -201,10 +304,13 @@ class MeowTests(unittest.TestCase):
         output_dir = os.path.join(TEST_JOB_OUTPUT, good_job_dict[JOB_ID])
         self.assertTrue(os.path.exists(output_dir))
         self.assertTrue(os.path.exists(os.path.join(output_dir, META_FILE)))
-        self.assertTrue(os.path.exists(os.path.join(output_dir, BASE_FILE)))
+        self.assertTrue(os.path.exists(
+            os.path.join(output_dir, get_base_file(JOB_TYPE_PAPERMILL))))
         self.assertTrue(os.path.exists(os.path.join(output_dir, PARAMS_FILE)))
-        self.assertTrue(os.path.exists(os.path.join(output_dir, JOB_FILE)))
+        self.assertTrue(os.path.exists(
-        self.assertTrue(os.path.exists(os.path.join(output_dir, RESULT_FILE)))
+            os.path.join(output_dir, get_job_file(JOB_TYPE_PAPERMILL))))
+        self.assertTrue(os.path.exists(
+            os.path.join(output_dir, get_result_file(JOB_TYPE_PAPERMILL))))
 
         self.assertTrue(os.path.exists(result_path))
 
@@ -235,7 +341,7 @@ class MeowTests(unittest.TestCase):
         rule = create_rule(pattern, recipe)
 
         job_dict = create_job(
-            JOB_TYPE_PYTHON,
+            JOB_TYPE_PAPERMILL,
             create_watchdog_event(
                 file_path,
                 rule,

tests/test_functionality.py

@@ -12,7 +12,7 @@ from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
     WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
     PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, JOB_EVENT, \
     JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
-    JOB_REQUIREMENTS, STATUS_QUEUED
+    JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
 from core.functionality import generate_id, wait, get_file_hash, rmtree, \
     make_dir, parameterize_jupyter_notebook, create_event, create_job, \
     replace_keywords, write_yaml, write_notebook, read_yaml, read_notebook, \
@@ -240,6 +240,8 @@ class CorrectnessTests(unittest.TestCase):
             pn["cells"][0]["source"], 
             "# The first cell\n\ns = 4\nnum = 1000")
 
+    # TODO Test  that parameterize_python_script parameterises given script
+
     # Test that create_event produces valid event dictionary
     def testCreateEvent(self)->None:
         pattern = FileEventPattern(
@@ -307,7 +309,7 @@ class CorrectnessTests(unittest.TestCase):
         )
 
         job_dict = create_job(
-            JOB_TYPE_PYTHON,
+            JOB_TYPE_PAPERMILL,
             event,
             extras={
                 JOB_PARAMETERS:{
@@ -328,7 +330,7 @@ class CorrectnessTests(unittest.TestCase):
         self.assertIn(JOB_EVENT, job_dict)
         self.assertEqual(job_dict[JOB_EVENT], event)
         self.assertIn(JOB_TYPE, job_dict)
-        self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PYTHON)
+        self.assertEqual(job_dict[JOB_TYPE], JOB_TYPE_PAPERMILL)
         self.assertIn(JOB_PATTERN, job_dict)
         self.assertEqual(job_dict[JOB_PATTERN], pattern.name)
         self.assertIn(JOB_RECIPE, job_dict)
@@ -659,3 +661,8 @@ class CorrectnessTests(unittest.TestCase):
         self.assertEqual(event[EVENT_RULE], rule)
         self.assertEqual(event["a"], 1)
         self.assertEqual(event[WATCHDOG_BASE], "base")
+
+#TODO test read file
+#TODO test readlines file
+#TODO test write file
+#TODO test lines to str

tests/test_meow.py

@@ -3,6 +3,7 @@ import unittest
  
 from typing import Any, Union, Tuple
 
+from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
 from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
     BaseHandler, BaseConductor, create_rules, create_rule
 from patterns import FileEventPattern
@@ -70,6 +71,78 @@ class MeowTests(unittest.TestCase):
                 pass
         FullPattern("name", "", "", "", "")
 
+    # Test expansion of parameter sweeps
+    def testBasePatternExpandSweeps(self)->None:
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one", sweep={
+                "s1":{
+                    SWEEP_START: 10, SWEEP_STOP: 20, SWEEP_JUMP:5
+                }
+            })
+
+        es = pattern_one.expand_sweeps()
+
+        self.assertIsInstance(es, list)
+        self.assertEqual(len(es), 3)
+
+        values = [
+            "s1-10", "s1-15", "s1-20", 
+        ]
+
+        for sweep_vals in es:
+            self.assertIsInstance(sweep_vals, tuple)
+            self.assertEqual(len(sweep_vals), 1)
+
+            val1 = None
+            for sweep_val in sweep_vals:
+                self.assertIsInstance(sweep_val, tuple)
+                self.assertEqual(len(sweep_val), 2)
+                if sweep_val[0] == "s1":
+                    val1 = f"s1-{sweep_val[1]}"
+            if val1:
+                values.remove(val1)
+        self.assertEqual(len(values), 0)
+
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one", sweep={
+                "s1":{
+                    SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
+                },
+                "s2":{
+                    SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
+                }
+            })
+
+        es = pattern_one.expand_sweeps()
+
+        self.assertIsInstance(es, list)
+        self.assertEqual(len(es), 15)
+
+        values = [
+            "s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20", 
+            "s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35", 
+            "s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50", 
+            "s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65", 
+            "s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80", 
+        ]
+
+        for sweep_vals in es:
+            self.assertIsInstance(sweep_vals, tuple)
+            self.assertEqual(len(sweep_vals), 2)
+
+            val1 = None
+            val2 = None
+            for sweep_val in sweep_vals:
+                self.assertIsInstance(sweep_val, tuple)
+                self.assertEqual(len(sweep_val), 2)
+                if sweep_val[0] == "s1":
+                    val1 = f"s1-{sweep_val[1]}"
+                if sweep_val[0] == "s2":
+                    val2 = f"s2-{sweep_val[1]}"
+            if val1 and val2:
+                values.remove(f"{val1}/{val2}")
+        self.assertEqual(len(values), 0)
+
     # Test that BaseRecipe instantiation
     def testBaseRule(self)->None:
         with self.assertRaises(TypeError):
@@ -87,7 +160,7 @@ class MeowTests(unittest.TestCase):
                 pass
             def _is_valid_pattern(self, pattern:Any)->None:
                 pass
-        FullRule("name", "", "")
+        FullRule("name", valid_pattern_one, valid_recipe_one)
 
     # Test that create_rule creates a rule from pattern and recipe
     def testCreateRule(self)->None:
@@ -227,5 +300,3 @@ class MeowTests(unittest.TestCase):
                 pass
 
         FullTestConductor()
-
-    # TODO Test expansion of parameter sweeps

tests/test_recipes.py

@@ -7,20 +7,25 @@ from multiprocessing import Pipe
 
 from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
     EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
-    JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
+    JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
-    PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, \
+    PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, META_FILE, JOB_ERROR, \
-    PARAMS_FILE, JOB_FILE, RESULT_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START
+    PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
+    get_base_file, get_job_file, get_result_file
 from core.correctness.validation import valid_job
 from core.functionality import get_file_hash, create_job, \
-    create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml
+    create_watchdog_event, make_dir, write_yaml, write_notebook, read_yaml, \
+    write_file, lines_to_string
 from core.meow import create_rules, create_rule
 from patterns.file_event_pattern import FileEventPattern
 from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
-    PapermillHandler, job_func
+    PapermillHandler, papermill_job_func
-from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
+from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
-from shared import setup, teardown, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
+from rules import FileEventJupyterNotebookRule, FileEventPythonRule
+from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
+    COMPLETE_PYTHON_SCRIPT, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
     TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK
 
+
 class JupyterNotebookTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
@@ -349,7 +354,7 @@ class JupyterNotebookTests(unittest.TestCase):
         }
 
         job_dict = create_job(
-            JOB_TYPE_PYTHON,
+            JOB_TYPE_PAPERMILL,
             create_watchdog_event(
                 file_path,
                 rule,
@@ -359,7 +364,7 @@ class JupyterNotebookTests(unittest.TestCase):
             extras={
                 JOB_PARAMETERS:params_dict,
                 JOB_HASH: file_hash,
-                PYTHON_FUNC:job_func,
+                PYTHON_FUNC:papermill_job_func,
                 PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
                 PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
             }
@@ -375,25 +380,35 @@ class JupyterNotebookTests(unittest.TestCase):
         param_file = os.path.join(job_dir, PARAMS_FILE)
         write_yaml(params_dict, param_file)
 
-        base_file = os.path.join(job_dir, BASE_FILE)
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
         write_notebook(APPENDING_NOTEBOOK, base_file)
 
-        job_func(job_dict)
+        papermill_job_func(job_dict)
 
         job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
         self.assertTrue(os.path.exists(job_dir))
-        self.assertTrue(os.path.exists(os.path.join(job_dir, META_FILE)))
+
-        self.assertTrue(os.path.exists(os.path.join(job_dir, BASE_FILE)))
+        meta_path = os.path.join(job_dir, META_FILE)
+        self.assertTrue(os.path.exists(meta_path))
+        status = read_yaml(meta_path)
+        self.assertIsInstance(status, dict)
+        self.assertIn(JOB_STATUS, status)
+        self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
+
+        self.assertTrue(os.path.exists(
+            os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))))
         self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE)))
-        self.assertTrue(os.path.exists(os.path.join(job_dir, JOB_FILE)))
+        self.assertTrue(os.path.exists(
-        self.assertTrue(os.path.exists(os.path.join(job_dir, RESULT_FILE)))
+            os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL))))
+        self.assertTrue(os.path.exists(
+            os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL))))
 
         self.assertTrue(os.path.exists(result_path))
 
     # Test jobFunc doesn't execute with no args
     def testJobFuncBadArgs(self)->None:
         try:
-            job_func({})
+            papermill_job_func({})
         except Exception:
             pass
 
@@ -411,3 +426,363 @@ class PythonTests(unittest.TestCase):
     def tearDown(self)->None:
         super().tearDown()
         teardown()
+ 
+    # Test PythonRecipe can be created
+    def testPythonRecipeCreationMinimum(self)->None:
+        PythonRecipe("test_recipe", BAREBONES_PYTHON_SCRIPT)
+
+    # Test PythonRecipe cannot be created without name
+    def testPythonRecipeCreationNoName(self)->None:
+        with self.assertRaises(ValueError):
+            PythonRecipe("", BAREBONES_PYTHON_SCRIPT)
+
+    # Test PythonRecipe cannot be created with invalid name
+    def testPythonRecipeCreationInvalidName(self)->None:
+        with self.assertRaises(ValueError):
+            PythonRecipe("@test_recipe", BAREBONES_PYTHON_SCRIPT)
+
+    # Test PythonRecipe cannot be created with invalid recipe
+    def testPythonRecipeCreationInvalidRecipe(self)->None:
+        with self.assertRaises(TypeError):
+            PythonRecipe("test_recipe", BAREBONES_NOTEBOOK)
+
+    # Test PythonRecipe name setup correctly
+    def testPythonRecipeSetupName(self)->None:
+        name = "name"
+        pr = PythonRecipe(name, BAREBONES_PYTHON_SCRIPT)
+        self.assertEqual(pr.name, name)
+
+    # Test PythonRecipe recipe setup correctly
+    def testPythonRecipeSetupRecipe(self)->None:
+        pr = PythonRecipe("name", BAREBONES_PYTHON_SCRIPT)
+        self.assertEqual(pr.recipe, BAREBONES_PYTHON_SCRIPT)
+
+    # Test PythonRecipe parameters setup correctly
+    def testPythonRecipeSetupParameters(self)->None:
+        parameters = {
+            "a": 1,
+            "b": True
+        }
+        pr = PythonRecipe(
+            "name", BAREBONES_PYTHON_SCRIPT, parameters=parameters)
+        self.assertEqual(pr.parameters, parameters)
+
+    # Test PythonRecipe requirements setup correctly
+    def testPythonRecipeSetupRequirements(self)->None:
+        requirements = {
+            "a": 1,
+            "b": True
+        }
+        pr = PythonRecipe(
+            "name", BAREBONES_PYTHON_SCRIPT, requirements=requirements)
+        self.assertEqual(pr.requirements, requirements)
+
+    # Test PythonHandler can be created
+    def testPythonHandlerMinimum(self)->None:
+        PythonHandler(
+            TEST_HANDLER_BASE, 
+            TEST_JOB_OUTPUT
+        )
+
+    # Test PythonHandler will handle given events
+    def testPythonHandlerHandling(self)->None:
+        from_handler_reader, from_handler_writer = Pipe()
+        ph = PythonHandler(
+            TEST_HANDLER_BASE,
+            TEST_JOB_OUTPUT
+        )
+        ph.to_runner = from_handler_writer
+        
+        with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
+            f.write("Data")
+
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one")
+        recipe = PythonRecipe(
+            "recipe_one", COMPLETE_PYTHON_SCRIPT)
+
+        patterns = {
+            pattern_one.name: pattern_one,
+        }
+        recipes = {
+            recipe.name: recipe,
+        }
+
+        rules = create_rules(patterns, recipes)
+        self.assertEqual(len(rules), 1)
+        _, rule = rules.popitem()
+        self.assertIsInstance(rule, FileEventPythonRule)
+
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+        event = {
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
+            WATCHDOG_BASE: TEST_MONITOR_BASE,
+            EVENT_RULE: rule,
+            WATCHDOG_HASH: get_file_hash(
+                os.path.join(TEST_MONITOR_BASE, "A"), SHA256
+            )
+        }
+
+        ph.handle(event)
+
+        if from_handler_reader.poll(3):
+            job_dir = from_handler_reader.recv()
+
+        self.assertIsInstance(job_dir, str)
+        self.assertTrue(os.path.exists(job_dir))
+
+        job = read_yaml(os.path.join(job_dir, META_FILE))
+        valid_job(job)
+
+    # Test PythonHandler will create enough jobs from single sweep
+    def testPythonHandlerHandlingSingleSweep(self)->None:
+        from_handler_reader, from_handler_writer = Pipe()
+        ph = PythonHandler(
+            TEST_HANDLER_BASE,
+            TEST_JOB_OUTPUT
+        )
+        ph.to_runner = from_handler_writer
+        
+        with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
+            f.write("Data")
+
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one", sweep={"s":{
+                SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
+            }})
+        recipe = PythonRecipe(
+            "recipe_one", COMPLETE_PYTHON_SCRIPT)
+
+        patterns = {
+            pattern_one.name: pattern_one,
+        }
+        recipes = {
+            recipe.name: recipe,
+        }
+
+        rules = create_rules(patterns, recipes)
+        self.assertEqual(len(rules), 1)
+        _, rule = rules.popitem()
+        self.assertIsInstance(rule, FileEventPythonRule)
+
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+        event = {
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
+            WATCHDOG_BASE: TEST_MONITOR_BASE,
+            EVENT_RULE: rule,
+            WATCHDOG_HASH: get_file_hash(
+                os.path.join(TEST_MONITOR_BASE, "A"), SHA256
+            )
+        }
+
+        ph.handle(event)
+
+        jobs = []
+        recieving = True
+        while recieving:
+            if from_handler_reader.poll(3):
+                jobs.append(from_handler_reader.recv())
+            else:
+                recieving = False
+
+        values = [0, 1, 2]
+        self.assertEqual(len(jobs), 3)
+        for job_dir in jobs:
+            self.assertIsInstance(job_dir, str)
+            self.assertTrue(os.path.exists(job_dir))
+
+            job = read_yaml(os.path.join(job_dir, META_FILE))
+            valid_job(job)
+
+            self.assertIn(JOB_PARAMETERS, job)
+            self.assertIn("s", job[JOB_PARAMETERS])
+            if job[JOB_PARAMETERS]["s"] in values:
+                values.remove(job[JOB_PARAMETERS]["s"])
+        self.assertEqual(len(values), 0)
+
+    # Test PythonHandler will create enough jobs from multiple sweeps
+    def testPythonHandlerHandlingMultipleSweep(self)->None:
+        from_handler_reader, from_handler_writer = Pipe()
+        ph = PythonHandler(
+            TEST_HANDLER_BASE,
+            TEST_JOB_OUTPUT
+        )
+        ph.to_runner = from_handler_writer
+        
+        with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
+            f.write("Data")
+
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one", sweep={
+                "s1":{
+                    SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
+                },
+                "s2":{
+                    SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
+                }
+            })
+        recipe = PythonRecipe(
+            "recipe_one", COMPLETE_PYTHON_SCRIPT)
+
+        patterns = {
+            pattern_one.name: pattern_one,
+        }
+        recipes = {
+            recipe.name: recipe,
+        }
+
+        rules = create_rules(patterns, recipes)
+        self.assertEqual(len(rules), 1)
+        _, rule = rules.popitem()
+        self.assertIsInstance(rule, FileEventPythonRule)
+
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+        event = {
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
+            WATCHDOG_BASE: TEST_MONITOR_BASE,
+            EVENT_RULE: rule,
+            WATCHDOG_HASH: get_file_hash(
+                os.path.join(TEST_MONITOR_BASE, "A"), SHA256
+            )
+        }
+
+        ph.handle(event)
+
+        jobs = []
+        recieving = True
+        while recieving:
+            if from_handler_reader.poll(3):
+                jobs.append(from_handler_reader.recv())
+            else:
+                recieving = False
+
+        values = [
+            "s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20", 
+            "s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35", 
+            "s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50", 
+            "s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65", 
+            "s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80", 
+        ]
+        self.assertEqual(len(jobs), 15)
+        for job_dir in jobs:
+            self.assertIsInstance(job_dir, str)
+            self.assertTrue(os.path.exists(job_dir))
+
+            job = read_yaml(os.path.join(job_dir, META_FILE))
+            valid_job(job)
+
+            self.assertIn(JOB_PARAMETERS, job)
+            val1 = None
+            val2 = None
+            if "s1" in job[JOB_PARAMETERS]:
+                val1 = f"s1-{job[JOB_PARAMETERS]['s1']}"
+            if "s2" in job[JOB_PARAMETERS]:
+                val2 = f"s2-{job[JOB_PARAMETERS]['s2']}"
+            val = None
+            if val1 and val2:
+                val = f"{val1}/{val2}"
+            if val and val in values:
+                values.remove(val)
+        self.assertEqual(len(values), 0)
+
+    # Test jobFunc performs as expected
+    def testJobFunc(self)->None:
+        file_path = os.path.join(TEST_MONITOR_BASE, "test")
+        result_path = os.path.join(TEST_MONITOR_BASE, "output")
+
+        with open(file_path, "w") as f:
+            f.write("250")
+
+        file_hash = get_file_hash(file_path, SHA256)
+
+        pattern = FileEventPattern(
+            "pattern", 
+            file_path, 
+            "recipe_one", 
+            "infile", 
+            parameters={
+                "extra":"A line from a test Pattern",
+                "outfile": result_path
+            })
+        recipe = PythonRecipe(
+            "recipe_one", COMPLETE_PYTHON_SCRIPT)
+
+        rule = create_rule(pattern, recipe)
+
+        params_dict = {
+            "extra":"extra",
+            "infile":file_path,
+            "outfile": result_path
+        }
+
+        job_dict = create_job(
+            JOB_TYPE_PYTHON,
+            create_watchdog_event(
+                file_path,
+                rule,
+                TEST_MONITOR_BASE,
+                file_hash
+            ),
+            extras={
+                JOB_PARAMETERS:params_dict,
+                JOB_HASH: file_hash,
+                PYTHON_FUNC:python_job_func,
+                PYTHON_OUTPUT_DIR:TEST_JOB_OUTPUT,
+                PYTHON_EXECUTION_BASE:TEST_HANDLER_BASE
+            }
+        )
+
+        job_dir = os.path.join(
+            job_dict[PYTHON_EXECUTION_BASE], job_dict[JOB_ID])
+        make_dir(job_dir)
+
+        meta_file = os.path.join(job_dir, META_FILE)
+        write_yaml(job_dict, meta_file)
+
+        param_file = os.path.join(job_dir, PARAMS_FILE)
+        write_yaml(params_dict, param_file)
+
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
+        write_notebook(APPENDING_NOTEBOOK, base_file)
+        write_file(lines_to_string(COMPLETE_PYTHON_SCRIPT), base_file)
+
+        python_job_func(job_dict)
+
+        job_dir = os.path.join(TEST_HANDLER_BASE, job_dict[JOB_ID])
+        self.assertTrue(os.path.exists(job_dir))
+        meta_path = os.path.join(job_dir, META_FILE)
+        self.assertTrue(os.path.exists(meta_path))
+
+        status = read_yaml(meta_path)
+        self.assertIsInstance(status, dict)
+        self.assertIn(JOB_STATUS, status)
+        self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])    
+        self.assertNotIn(JOB_ERROR, status)
+
+        self.assertTrue(os.path.exists(
+            os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))))
+        self.assertTrue(os.path.exists(os.path.join(job_dir, PARAMS_FILE)))
+        self.assertTrue(os.path.exists(
+            os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON))))
+        self.assertTrue(os.path.exists(
+            os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON))))
+
+        self.assertTrue(os.path.exists(result_path))
+
+    # Test jobFunc doesn't execute with no args
+    def testJobFuncBadArgs(self)->None:
+        try:
+            python_job_func({})
+        except Exception:
+            pass
+
+        self.assertEqual(len(os.listdir(TEST_HANDLER_BASE)), 0)
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+    # TODO test default parameter function execution

tests/test_runner.py

@@ -6,7 +6,7 @@ import unittest
 from time import sleep
 
 from conductors import LocalPythonConductor
-from core.correctness.vars import RESULT_FILE
+from core.correctness.vars import get_result_file, JOB_TYPE_PAPERMILL
 from core.functionality import make_dir, read_notebook
 from core.meow import BaseMonitor, BaseHandler, BaseConductor
 from core.runner import MeowRunner
@@ -187,7 +187,8 @@ class MeowTests(unittest.TestCase):
         job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
         self.assertEqual(len(os.listdir(job_dir)), 5)
 
-        result = read_notebook(os.path.join(job_dir, RESULT_FILE))
+        result = read_notebook(
+            os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
         self.assertIsNotNone(result)
 
         output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
@@ -279,7 +280,8 @@ class MeowTests(unittest.TestCase):
         mid_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
         self.assertEqual(len(os.listdir(mid_job_dir)), 5)
 
-        result = read_notebook(os.path.join(mid_job_dir, RESULT_FILE))
+        result = read_notebook(
+            os.path.join(mid_job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
         self.assertIsNotNone(result)
 
         mid_output_path = os.path.join(TEST_MONITOR_BASE, "middle", "A.txt")
@@ -293,7 +295,8 @@ class MeowTests(unittest.TestCase):
         final_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
         self.assertEqual(len(os.listdir(final_job_dir)), 5)
 
-        result = read_notebook(os.path.join(final_job_dir, RESULT_FILE))
+        result = read_notebook(os.path.join(final_job_dir, 
+            get_result_file(JOB_TYPE_PAPERMILL)))
         self.assertIsNotNone(result)
 
         final_output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
@@ -305,8 +308,8 @@ class MeowTests(unittest.TestCase):
         self.assertEqual(data, 
             "Initial Data\nA line from Pattern 1\nA line from Pattern 2")
 
-    # TODO sweep tests
+    # TODO sweep execution test
-    # TODO adding tests with numpy
+    # TODO adding tests with numpy or other external dependency
     # TODO test getting job cannot handle
     # TODO test getting event cannot handle
     # TODO test with several matched monitors

tests/test_validation.py

@@ -9,7 +9,7 @@ from typing import Any, Union
 from core.correctness.validation import check_type, check_implementation, \
     valid_string, valid_dict, valid_list, valid_existing_file_path, \
     valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
-    setup_debugging, valid_watchdog_event
+    setup_debugging, valid_watchdog_event, check_callable
 from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
     EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
     JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
@@ -304,7 +304,7 @@ class CorrectnessTests(unittest.TestCase):
         with self.assertRaises(TypeError):
             setup_debugging(stream, "1")
 
-    #Test watchdog event dict
+    # Test watchdog event dict
     def testWatchdogEventValidation(self)->None:
         valid_watchdog_event({
             EVENT_TYPE: "test", 
@@ -337,3 +337,10 @@ class CorrectnessTests(unittest.TestCase):
 
         with self.assertRaises(KeyError):
             valid_event({})
+
+    # Test check_callable 
+    def testCheckCallable(self)->None:
+        check_callable(make_dir)
+        
+        with self.assertRaises(TypeError):
+            check_callable("a")