rewored rules to only invoke base rule, and added bash jobs

conductors/local_bash_conductor.py

@@ -0,0 +1,129 @@
+
+"""
+This file contains definitions for the LocalBashConductor, in order to 
+execute Bash jobs on the local resource.
+
+Author(s): David Marchant
+"""
+
+import os
+import shutil
+import subprocess
+
+from datetime import datetime
+from typing import Any, Dict, Tuple
+
+from meow_base.core.base_conductor import BaseConductor
+from meow_base.core.correctness.meow import valid_job
+from meow_base.core.correctness.vars import DEFAULT_JOB_QUEUE_DIR, \
+    DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, META_FILE, JOB_STATUS, \
+    BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
+    JOB_ERROR, JOB_TYPE, DEFAULT_JOB_QUEUE_DIR, STATUS_RUNNING, \
+    JOB_START_TIME, DEFAULT_JOB_OUTPUT_DIR, get_job_file
+from meow_base.core.correctness.validation import valid_dir_path
+from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
+    write_yaml
+
+
+class LocalBashConductor(BaseConductor):
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
+            job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="")->None:
+        """LocalBashConductor Constructor. This should be used to execute 
+        Bash jobs, and will then pass any internal job runner files to the 
+        output directory. Note that if this handler is given to a MeowRunner
+        object, the job_queue_dir and job_output_dir will be overwridden."""
+        super().__init__(name=name)
+        self._is_valid_job_queue_dir(job_queue_dir)
+        self.job_queue_dir = job_queue_dir
+        self._is_valid_job_output_dir(job_output_dir)
+        self.job_output_dir = job_output_dir
+
+    def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
+        """Function to determine given an job defintion, if this conductor can 
+        process it or not. This conductor will accept any Bash job type"""
+        try:
+            valid_job(job)
+            msg = ""
+            if job[JOB_TYPE] not in [JOB_TYPE_BASH]:
+                msg = f"Job type was not {JOB_TYPE_BASH}."
+            if msg:
+                return False, msg
+            else:
+                return True, ""
+        except Exception as e:
+            return False, str(e)
+
+    def execute(self, job_dir:str)->None:
+        """Function to actually execute a Bash job. This will read job 
+        defintions from its meta file, update the meta file and attempt to 
+        execute. Some unspecific feedback will be given on execution failure, 
+        but depending on what it is it may be up to the job itself to provide 
+        more detailed feedback."""
+        valid_dir_path(job_dir, must_exist=True)
+
+        # Test our job parameters. Even if its gibberish, we still move to 
+        # output
+        abort = False
+        try:
+            meta_file = os.path.join(job_dir, META_FILE)
+            job = read_yaml(meta_file)
+            valid_job(job)
+
+            # update the status file with running status
+            job[JOB_STATUS] = STATUS_RUNNING
+            job[JOB_START_TIME] = datetime.now()
+            write_yaml(job, meta_file)
+
+        except Exception as e:
+            # If something has gone wrong at this stage then its bad, so we 
+            # need to make our own error file
+            error_file = os.path.join(job_dir, BACKUP_JOB_ERROR_FILE)
+            write_file(f"Recieved incorrectly setup job.\n\n{e}", error_file)
+            abort = True
+
+        # execute the job
+        if not abort:
+            try:
+                result = subprocess.call(get_job_file(JOB_TYPE_BASH), cwd=".")
+
+                # get up to date job data
+                job = read_yaml(meta_file)
+
+                # Update the status file with the finalised status
+                job[JOB_STATUS] = STATUS_DONE
+                job[JOB_END_TIME] = datetime.now()
+                write_yaml(job, meta_file)
+
+            except Exception as e:
+                # get up to date job data
+                job = read_yaml(meta_file)
+
+                # Update the status file with the error status. Don't overwrite
+                # any more specific error messages already created
+                if JOB_STATUS not in job:
+                    job[JOB_STATUS] = STATUS_FAILED
+                if JOB_END_TIME not in job:
+                    job[JOB_END_TIME] = datetime.now()
+                if JOB_ERROR not in job:
+                    job[JOB_ERROR] = f"Job execution failed. {e}"
+                write_yaml(job, meta_file)
+
+        # Move the contents of the execution directory to the final output 
+        # directory. 
+        job_output_dir = \
+            os.path.join(self.job_output_dir, os.path.basename(job_dir))
+        shutil.move(job_dir, job_output_dir)
+
+    def _is_valid_job_queue_dir(self, job_queue_dir)->None:
+        """Validation check for 'job_queue_dir' variable from main 
+        constructor."""
+        valid_dir_path(job_queue_dir, must_exist=False)
+        if not os.path.exists(job_queue_dir):
+            make_dir(job_queue_dir)
+
+    def _is_valid_job_output_dir(self, job_output_dir)->None:
+        """Validation check for 'job_output_dir' variable from main 
+        constructor."""
+        valid_dir_path(job_output_dir, must_exist=False)
+        if not os.path.exists(job_output_dir):
+            make_dir(job_output_dir)

core/base_monitor.py

@@ -11,7 +11,7 @@ from typing import Union, Dict
 
 from meow_base.core.base_pattern import BasePattern
 from meow_base.core.base_recipe import BaseRecipe
-from meow_base.core.base_rule import BaseRule
+from meow_base.core.rule import Rule
 from meow_base.core.correctness.vars import VALID_CHANNELS, \
     VALID_MONITOR_NAME_CHARS, get_drt_imp_msg 
 from meow_base.core.correctness.validation import check_implementation, \
@@ -29,7 +29,7 @@ class BaseMonitor:
     # A collection of recipes
     _recipes: Dict[str, BaseRecipe]
     # A collection of rules derived from _patterns and _recipes
-    _rules: Dict[str, BaseRule]
+    _rules: Dict[str, Rule]
     # A channel for sending messages to the runner. Note that this is not 
     # initialised within the constructor, but within the runner when passed the
     # monitor is passed to it.
@@ -138,7 +138,7 @@ class BaseMonitor:
         Must be implemented by any child process."""
         pass
 
-    def get_rules(self)->Dict[str,BaseRule]:
+    def get_rules(self)->Dict[str,Rule]:
         """Function to get a dictionary of all current rule definitions. 
         Must be implemented by any child process."""
         pass

core/base_rule.py

@@ -1,85 +0,0 @@
-
-"""
-This file contains the base MEOW rule defintion. This should be inherited from 
-for all rule instances.
-
-Author(s): David Marchant
-"""
-
-from sys import modules
-from typing import Any
-
-if "BasePattern" not in modules:
-    from meow_base.core.base_pattern import BasePattern
-if "BaseRecipe" not in modules:
-    from meow_base.core.base_recipe import BaseRecipe
-from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \
-    get_drt_imp_msg
-from meow_base.core.correctness.validation import valid_string, check_type, \
-    check_implementation
-
-
-class BaseRule:
-    # A unique identifier for the rule
-    name:str
-    # A pattern to be used in rule triggering
-    pattern:BasePattern
-    # A recipe to be used in rule execution
-    recipe:BaseRecipe
-    # The string name of the pattern class that can be used to create this rule
-    pattern_type:str=""
-    # The string name of the recipe class that can be used to create this rule
-    recipe_type:str=""
-    def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
-        """BaseRule Constructor. This will check that any class inheriting 
-        from it implements its validation functions. It will then call these on
-        the input parameters."""
-        check_implementation(type(self)._is_valid_pattern, BaseRule)
-        check_implementation(type(self)._is_valid_recipe, BaseRule)
-        self.__check_types_set()
-        self._is_valid_name(name)
-        self.name = name
-        self._is_valid_pattern(pattern)
-        self.pattern = pattern
-        self._is_valid_recipe(recipe)
-        self.recipe = recipe
-        check_type(pattern, BasePattern, hint="BaseRule.pattern")
-        check_type(recipe, BaseRecipe, hint="BaseRule.recipe")
-        if pattern.recipe != recipe.name:
-            raise ValueError(f"Cannot create Rule {name}. Pattern "
-                f"{pattern.name} does not identify Recipe {recipe.name}. It "
-                f"uses {pattern.recipe}")
-
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BaseRule:
-            msg = get_drt_imp_msg(BaseRule)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def _is_valid_name(self, name:str)->None:
-        """Validation check for 'name' variable from main constructor. Is 
-        automatically called during initialisation. This does not need to be 
-        overridden by child classes."""
-        valid_string(name, VALID_RULE_NAME_CHARS)
-
-    def _is_valid_pattern(self, pattern:Any)->None:
-        """Validation check for 'pattern' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def _is_valid_recipe(self, recipe:Any)->None:
-        """Validation check for 'recipe' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def __check_types_set(self)->None:
-        """Validation check that the self.pattern_type and self.recipe_type 
-        attributes have been set in a child class."""
-        if self.pattern_type == "":
-            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
-                "does not set a pattern_type.")
-        if self.recipe_type == "":
-            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
-                "does not set a recipe_type.")

core/correctness/meow.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import Any, Dict, Type
 
-from meow_base.core.base_rule import BaseRule
+from meow_base.core.rule import Rule
 from meow_base.core.correctness.validation import check_type
 from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \
     JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
@@ -13,7 +13,7 @@ EVENT_KEYS = {
     EVENT_TYPE: str,
     EVENT_PATH: str,
     # Should be a Rule but can't import here due to circular dependencies
-    EVENT_RULE: BaseRule
+    EVENT_RULE: Rule
 }
 
 WATCHDOG_EVENT_KEYS = {

core/correctness/vars.py

@@ -86,6 +86,7 @@ DEFAULT_JOB_OUTPUT_DIR = "job_output"
 
 # meow jobs
 JOB_TYPE = "job_type"
+JOB_TYPE_BASH = "bash"
 JOB_TYPE_PYTHON = "python"
 JOB_TYPE_PAPERMILL = "papermill"
 PYTHON_FUNC = "func"
@@ -94,12 +95,17 @@ JOB_TYPES = {
     JOB_TYPE_PAPERMILL: [
         "base.ipynb",
         "job.ipynb",
-        "result.ipynb",
+        "result.ipynb"
     ],
     JOB_TYPE_PYTHON: [
         "base.py",
         "job.py",
-        "result.py",
+        "result.py"
+    ],
+    JOB_TYPE_BASH: [
+        "base.sh",
+        "job.sh",
+        "result.sh"
     ]
 }
 

core/rule.py

@@ -0,0 +1,49 @@
+
+"""
+This file contains the MEOW rule defintion.
+
+Author(s): David Marchant
+"""
+
+from sys import modules
+from typing import Any
+
+if "BasePattern" not in modules:
+    from meow_base.core.base_pattern import BasePattern
+if "BaseRecipe" not in modules:
+    from meow_base.core.base_recipe import BaseRecipe
+from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \
+    get_drt_imp_msg
+from meow_base.core.correctness.validation import valid_string, check_type, \
+    check_implementation
+from meow_base.functionality.naming import generate_rule_id
+
+
+class Rule:
+    # A unique identifier for the rule
+    name:str
+    # A pattern to be used in rule triggering
+    pattern:BasePattern
+    # A recipe to be used in rule execution
+    recipe:BaseRecipe
+    def __init__(self, pattern:BasePattern, recipe:BaseRecipe, name:str=""):
+        """Rule Constructor. This will check that any class inheriting 
+        from it implements its validation functions. It will then call these on
+        the input parameters."""
+        if not name:
+            name = generate_rule_id()
+        self._is_valid_name(name)
+        self.name = name
+        check_type(pattern, BasePattern, hint="Rule.pattern")
+        self.pattern = pattern
+        check_type(recipe, BaseRecipe, hint="Rule.recipe")
+        self.recipe = recipe
+        if pattern.recipe != recipe.name:
+            raise ValueError(f"Cannot create Rule {name}. Pattern "
+                f"{pattern.name} does not identify Recipe {recipe.name}. It "
+                f"uses {pattern.recipe}")
+
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation."""
+        valid_string(name, VALID_RULE_NAME_CHARS)

functionality/meow.py

@@ -10,7 +10,7 @@ from typing import Any, Dict, Union, List
 
 from meow_base.core.base_pattern import BasePattern
 from meow_base.core.base_recipe import BaseRecipe
-from meow_base.core.base_rule import BaseRule
+from meow_base.core.rule import Rule
 from meow_base.core.correctness.validation import check_type, valid_dict, \
     valid_list
 from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \
@@ -18,7 +18,7 @@ from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \
     JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
     JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \
     SWEEP_START, SWEEP_STOP
-from meow_base.functionality.naming import generate_job_id, generate_rule_id
+from meow_base.functionality.naming import generate_job_id
 
 # mig trigger keyword replacements
 KEYWORD_PATH = "{PATH}"
@@ -147,8 +147,7 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
     return {**extras, **job_dict}
 
 def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], 
-        recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
+        recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]])->Dict[str,Rule]:
-        new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
     """Function to create any valid rules from a given collection of patterns 
     and recipes. All inbuilt rule types are considered, with additional 
     definitions provided through the 'new_rules' variable. Note that any 
@@ -157,7 +156,6 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
     # Validation of inputs
     check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns")
     check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes")
-    valid_list(new_rules, BaseRule, min_length=0)
 
     # Convert a pattern list to a dictionary
     if isinstance(patterns, list):
@@ -198,34 +196,14 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
                 pass
     return generated_rules
 
-def create_rule(pattern:BasePattern, recipe:BaseRecipe, 
+def create_rule(pattern:BasePattern, recipe:BaseRecipe)->Rule:
-        new_rules:List[BaseRule]=[])->BaseRule:
     """Function to create a valid rule from a given pattern and recipe. All 
     inbuilt rule types are considered, with additional definitions provided 
     through the 'new_rules' variable."""
     check_type(pattern, BasePattern, hint="create_rule.pattern")
     check_type(recipe, BaseRecipe, hint="create_rule.recipe")
-    valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules")
     
-    # TODO fix me
+    return Rule(
-    # Imported here to avoid circular imports at top of file
-    import meow_base.rules
-    all_rules = {
-        (r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()
-    }
-
-    # Add in new rules
-    for rule in new_rules:
-        all_rules[(rule.pattern_type, rule.recipe_type)] = rule
-
-    # Find appropriate rule type from pattern and recipe types
-    key = (type(pattern).__name__, type(recipe).__name__)
-    if (key) in all_rules:
-        return all_rules[key](
-            generate_rule_id(), 
         pattern, 
         recipe
     )
-    # Raise error if not valid rule type can be found
-    raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe "
-        f"'{recipe}' could be found.")

functionality/parameterisation.py

@@ -119,3 +119,36 @@ def parameterize_python_script(script:List[str], parameters:Dict[str,Any],
     check_script(output_script)
 
     return output_script
+
+def parameterize_bash_script(script:List[str], parameters:Dict[str,Any], 
+        expand_env_values:bool=False)->Dict[str,Any]:
+    check_script(script)
+    check_type(parameters, Dict
+        ,hint="parameterize_bash_script.parameters")
+
+    output_script = deepcopy(script)
+
+    for i, line in enumerate(output_script):
+        if "=" in line:
+            d_line = list(map(lambda x: x.replace(" ", ""), 
+                line.split("=")))
+            # Matching parameter name
+            if len(d_line) == 2 and d_line[0] in parameters:
+                value = parameters[d_line[0]]
+                # Whether to expand value from os env
+                if (
+                    expand_env_values
+                    and isinstance(value, str)
+                    and value.startswith("ENV_")
+                ):
+                    env_var = value.replace("ENV_", "")
+                    value = getenv(
+                        env_var, 
+                        "MISSING ENVIRONMENT VARIABLE: {}".format(env_var)
+                    )
+                output_script[i] = f"{d_line[0]}={repr(value)}"
+                
+    # Validate that the parameterized notebook is still valid
+    check_script(output_script)
+
+    return output_script

patterns/file_event_pattern.py

@@ -21,7 +21,7 @@ from watchdog.events import PatternMatchingEventHandler
 from meow_base.core.base_recipe import BaseRecipe
 from meow_base.core.base_monitor import BaseMonitor
 from meow_base.core.base_pattern import BasePattern
-from meow_base.core.base_rule import BaseRule
+from meow_base.core.rule import Rule
 from meow_base.core.correctness.validation import check_type, valid_string, \
     valid_dict, valid_list, valid_dir_path
 from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
@@ -358,7 +358,7 @@ class WatchdogMonitor(BaseMonitor):
         self._recipes_lock.release()
         return to_return
     
-    def get_rules(self)->Dict[str,BaseRule]:
+    def get_rules(self)->Dict[str,Rule]:
         """Function to get a dict of the currently defined rules of the 
         monitor. Note that the result is deep-copied, and so can be manipulated
         without directly manipulating the internals of the monitor."""
@@ -486,7 +486,7 @@ class WatchdogMonitor(BaseMonitor):
         for rule in self._rules.values():
             self._apply_retroactive_rule(rule)
 
-    def _apply_retroactive_rule(self, rule:BaseRule)->None:
+    def _apply_retroactive_rule(self, rule:Rule)->None:
         """Function to determine if a rule should be applied to the existing 
         file structure, were the file structure created/modified now."""
         self._rules_lock.acquire()

recipes/bash_recipe.py

@@ -0,0 +1,212 @@
+
+import os
+import stat
+import sys
+
+from typing import Any, Dict, List, Tuple
+
+from meow_base.core.base_handler import BaseHandler
+from meow_base.core.base_recipe import BaseRecipe
+from meow_base.core.correctness.meow import valid_event
+from meow_base.core.correctness.validation import check_type, valid_dict, \
+    valid_string, valid_dir_path
+from meow_base.core.correctness.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
+    VALID_VARIABLE_NAME_CHARS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, JOB_ID, \
+    EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_PARAMETERS, JOB_HASH, \
+    WATCHDOG_HASH, WATCHDOG_BASE, META_FILE, PARAMS_FILE, STATUS_QUEUED, \
+    JOB_STATUS, \
+    get_base_file, get_job_file
+from meow_base.functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.file_io import valid_path, make_dir, write_yaml, \
+    write_file, lines_to_string
+from meow_base.functionality.parameterisation import parameterize_bash_script
+from meow_base.functionality.meow import create_job, replace_keywords
+
+
+class BashRecipe(BaseRecipe):
+    # A path to the bash script used to create this recipe
+    def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, 
+            requirements:Dict[str,Any]={}, source:str=""):
+        """BashRecipe Constructor. This is used to execute bash scripts, 
+        enabling anything not natively supported by MEOW."""
+        super().__init__(name, recipe, parameters, requirements)
+        self._is_valid_source(source)
+        self.source = source
+
+    def _is_valid_source(self, source:str)->None:
+        """Validation check for 'source' variable from main constructor."""
+        if source:
+            valid_path(source, extension=".sh", min_length=0)
+
+    def _is_valid_recipe(self, recipe:List[str])->None:
+        """Validation check for 'recipe' variable from main constructor. 
+        Called within parent BaseRecipe constructor."""
+        check_type(recipe, List, hint="BashRecipe.recipe")
+        for line in recipe:
+            check_type(line, str, hint="BashRecipe.recipe[line]")
+
+    def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
+        """Validation check for 'parameters' variable from main constructor. 
+        Called within parent BaseRecipe constructor."""
+        valid_dict(parameters, str, Any, strict=False, min_length=0)
+        for k in parameters.keys():
+            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+
+    def _is_valid_requirements(self, requirements:Dict[str,Any])->None:
+        """Validation check for 'requirements' variable from main constructor. 
+        Called within parent BaseRecipe constructor."""
+        valid_dict(requirements, str, Any, strict=False, min_length=0)
+        for k in requirements.keys():
+            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+
+
+class BashHandler(BaseHandler):
+    # Config option, above which debug messages are ignored
+    debug_level:int
+    # Where print messages are sent
+    _print_target:Any
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
+            print:Any=sys.stdout, logging:int=0)->None:
+        """BashHandler Constructor. This creates jobs to be executed as 
+        bash scripts. This does not run as a continuous thread to 
+        handle execution, but is invoked according to a factory pattern using 
+        the handle function. Note that if this handler is given to a MeowRunner
+        object, the job_queue_dir will be overwridden by its"""
+        super().__init__(name=name)
+        self._is_valid_job_queue_dir(job_queue_dir)
+        self.job_queue_dir = job_queue_dir
+        self._print_target, self.debug_level = setup_debugging(print, logging)
+        print_debug(self._print_target, self.debug_level, 
+            "Created new BashHandler instance", DEBUG_INFO)
+
+    def handle(self, event:Dict[str,Any])->None:
+        """Function called to handle a given event."""
+        print_debug(self._print_target, self.debug_level, 
+            f"Handling event {event[EVENT_PATH]}", DEBUG_INFO)
+
+        rule = event[EVENT_RULE]
+
+        # Assemble job parameters dict from pattern variables
+        yaml_dict = {}
+        for var, val in rule.pattern.parameters.items():
+            yaml_dict[var] = val
+        for var, val in rule.pattern.outputs.items():
+            yaml_dict[var] = val
+        yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH]
+
+        # If no parameter sweeps, then one job will suffice
+        if not rule.pattern.sweep:
+            self.setup_job(event, yaml_dict)
+        else:
+            # If parameter sweeps, then many jobs created
+            values_list = rule.pattern.expand_sweeps()
+            for values in values_list:
+                for value in values:
+                    yaml_dict[value[0]] = value[1]
+                self.setup_job(event, yaml_dict)
+
+    def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
+        """Function to determine given an event defintion, if this handler can 
+        process it or not. This handler accepts events from watchdog with 
+        Bash recipes"""
+        try:
+            valid_event(event)
+            msg = ""
+            if type(event[EVENT_RULE].recipe) != BashRecipe:
+                msg = "Recipe is not a BashRecipe. "
+            if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG:
+                msg += f"Event type is not {EVENT_TYPE_WATCHDOG}."
+            if msg:
+                return False, msg
+            else:
+                return True, ""
+        except Exception as e:
+            return False, str(e)
+
+    def _is_valid_job_queue_dir(self, job_queue_dir)->None:
+        """Validation check for 'job_queue_dir' variable from main 
+        constructor."""
+        valid_dir_path(job_queue_dir, must_exist=False)
+        if not os.path.exists(job_queue_dir):
+            make_dir(job_queue_dir)
+
+    def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None:
+        """Function to set up new job dict and send it to the runner to be 
+        executed."""
+        meow_job = create_job(
+            JOB_TYPE_BASH, 
+            event, 
+            extras={
+                JOB_PARAMETERS:yaml_dict,
+                JOB_HASH: event[WATCHDOG_HASH],
+#                CONTROL_SCRIPT:python_job_func
+            }
+        )
+        print_debug(self._print_target, self.debug_level,  
+            f"Creating job from event at {event[EVENT_PATH]} of type "
+            f"{JOB_TYPE_BASH}.", DEBUG_INFO)
+
+        # replace MEOW keyworks within variables dict
+        yaml_dict = replace_keywords(
+            meow_job[JOB_PARAMETERS],
+            meow_job[JOB_ID],
+            event[EVENT_PATH],
+            event[WATCHDOG_BASE]
+        )
+
+        # Create a base job directory
+        job_dir = os.path.join(self.job_queue_dir, meow_job[JOB_ID])
+        make_dir(job_dir)
+
+        # write a status file to the job directory
+        meta_file = os.path.join(job_dir, META_FILE)
+        write_yaml(meow_job, meta_file)
+
+        # parameterise recipe and write as executeable script
+        base_script = parameterize_bash_script(
+            event[EVENT_RULE].recipe.recipe, yaml_dict
+        )
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))
+        write_file(lines_to_string(base_script), base_file)
+        os.chmod(base_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+        # Write job script, to manage base script lifetime and execution
+
+        job_script = assemble_bash_job_script()
+        job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))
+        write_file(lines_to_string(job_script), job_file)
+        os.chmod(job_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+        meow_job[JOB_STATUS] = STATUS_QUEUED
+
+        # update the status file with queued status
+        write_yaml(meow_job, meta_file)
+        
+        # Send job directory, as actual definitons will be read from within it
+        self.to_runner.send(job_dir)
+
+
+def assemble_bash_job_script()->List[str]:
+    return [
+        "#!/bin/bash",
+        "",
+        "# Get job params",
+        "given_hash=$(grep 'file_hash: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)",
+        "event_path=$(grep 'event_path: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)",
+        "",
+        "echo event_path: $event_path",
+        "echo given_hash: $given_hash",
+        "",
+        "# Check hash of input file to avoid race conditions",
+        "actual_hash=$(sha256sum $event_path | cut -c -64)",
+        "echo actual_hash: $actual_hash",
+        "if [ $given_hash != $actual_hash ]; then",
+        "   echo Job was skipped as triggering file has been modified since scheduling",
+        "   exit 134",
+        "fi",
+        "",
+        "# Call actual job script",
+        "$(dirname $0)/base.sh",
+        "",
+        "exit $?"
+    ]

recipes/python_recipe.py

@@ -65,7 +65,7 @@ class PythonHandler(BaseHandler):
         python functions. This does not run as a continuous thread to 
         handle execution, but is invoked according to a factory pattern using 
         the handle function. Note that if this handler is given to a MeowRunner
-        object, the job_queue_dir will be overwridden but its"""
+        object, the job_queue_dir will be overwridden by its"""
         super().__init__(name=name)
         self._is_valid_job_queue_dir(job_queue_dir)
         self.job_queue_dir = job_queue_dir

rules/__init__.py

@@ -1,3 +0,0 @@
-
-from .file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
-from .file_event_python_rule import FileEventPythonRule

rules/file_event_jupyter_notebook_rule.py

@@ -1,43 +0,0 @@
-
-"""
-This file contains definitions for a MEOW rule connecting the FileEventPattern
-and JupyterNotebookRecipe.
-
-Author(s): David Marchant
-"""
-
-from meow_base.core.base_rule import BaseRule
-from meow_base.core.correctness.validation import check_type
-from meow_base.patterns.file_event_pattern import FileEventPattern
-from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
-
-# TODO potentailly remove this and just invoke BaseRule directly, as does not
-# add any functionality other than some validation.
-class FileEventJupyterNotebookRule(BaseRule):
-    pattern_type = "FileEventPattern"
-    recipe_type = "JupyterNotebookRecipe" 
-    def __init__(self, name: str, pattern:FileEventPattern, 
-            recipe:JupyterNotebookRecipe):
-        super().__init__(name, pattern, recipe)
-        if pattern.recipe != recipe.name:
-            raise ValueError(f"Cannot create Rule {name}. Pattern "
-                f"{pattern.name} does not identify Recipe {recipe.name}. It "
-                f"uses {pattern.recipe}")
-
-    def _is_valid_pattern(self, pattern:FileEventPattern)->None:
-        """Validation check for 'pattern' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            pattern, 
-            FileEventPattern,
-            hint="FileEventJupyterNotebookRule.pattern"
-        )
-
-    def _is_valid_recipe(self, recipe:JupyterNotebookRecipe)->None:
-        """Validation check for 'recipe' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            recipe, 
-            JupyterNotebookRecipe,
-            hint="FileEventJupyterNotebookRule.recipe"
-        )

rules/file_event_python_rule.py

@@ -1,39 +0,0 @@
-
-"""
-This file contains definitions for a MEOW rule connecting the FileEventPattern
-and PythonRecipe.
-
-Author(s): David Marchant
-"""
-
-from meow_base.core.base_rule import BaseRule
-from meow_base.core.correctness.validation import check_type
-from meow_base.patterns.file_event_pattern import FileEventPattern
-from meow_base.recipes.python_recipe import PythonRecipe
-
-# TODO potentailly remove this and just invoke BaseRule directly, as does not
-# add any functionality other than some validation.
-class FileEventPythonRule(BaseRule):
-    pattern_type = "FileEventPattern"
-    recipe_type = "PythonRecipe" 
-    def __init__(self, name: str, pattern:FileEventPattern, 
-            recipe:PythonRecipe):
-        super().__init__(name, pattern, recipe)
-
-    def _is_valid_pattern(self, pattern:FileEventPattern)->None:
-        """Validation check for 'pattern' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            pattern, 
-            FileEventPattern,
-            hint="FileEventPythonRule.pattern"
-        )
-
-    def _is_valid_recipe(self, recipe:PythonRecipe)->None:
-        """Validation check for 'recipe' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            recipe, 
-            PythonRecipe,
-            hint="FileEventPythonRule.recipe"
-        )

tests/shared.py

@@ -46,34 +46,36 @@ def backup_before_teardown(backup_source:str, backup_dest:str):
     copy_tree(backup_source, backup_dest)
 
 
-# Recipe funcs
+# Bash scripts
-BAREBONES_PYTHON_SCRIPT = [
+BAREBONES_BASH_SCRIPT = [
     ""
 ]
-COMPLETE_PYTHON_SCRIPT = [
+COMPLETE_BASH_SCRIPT = [
-    "import os",
+    '#!/bin/bash',
-    "# Setup parameters",
+    '',
-    "num = 1000",
+    'num=1000',
-    "infile = 'somehere"+ os.path.sep +"particular'",
+    'infile="data"',
-    "outfile = 'nowhere"+ os.path.sep +"particular'",
+    'outfile="output"',
-    "",
+    '',
-    "with open(infile, 'r') as file:",
+    'echo "starting"',
-    "    s = float(file.read())",
+    '',
-    ""
+    'read var < $infile',
-    "for i in range(num):",
+    'for (( i=0; i<$num; i++ ))',
-    "    s += i",
+    'do',
-    "",
+    '    var=$((var+i))',
-    "div_by = 4",
+    'done',
-    "result = s / div_by",
+    '',
-    "",
+    'div_by=4',
-    "print(result)",
+    'echo $var',
-    "",
+    'result=$((var/div_by))',
-    "os.makedirs(os.path.dirname(outfile), exist_ok=True)",
+    '',
-    "",
+    'echo $result',
-    "with open(outfile, 'w') as file:",
+    '',
-    "    file.write(str(result))",
+    'mkdir -p $(dirname $outfile)',
-    "",
+    '',
-    "print('done')"
+    'echo $result > $outfile',
+    '',
+    'echo "done"'
 ]
 
 # Jupyter notebooks
@@ -1225,6 +1227,34 @@ GENERATOR_NOTEBOOK = {
 }
 
 # Python scripts
+BAREBONES_PYTHON_SCRIPT = [
+    ""
+]
+COMPLETE_PYTHON_SCRIPT = [
+    "import os",
+    "# Setup parameters",
+    "num = 1000",
+    "infile = 'somehere"+ os.path.sep +"particular'",
+    "outfile = 'nowhere"+ os.path.sep +"particular'",
+    "",
+    "with open(infile, 'r') as file:",
+    "    s = float(file.read())",
+    ""
+    "for i in range(num):",
+    "    s += i",
+    "",
+    "div_by = 4",
+    "result = s / div_by",
+    "",
+    "print(result)",
+    "",
+    "os.makedirs(os.path.dirname(outfile), exist_ok=True)",
+    "",
+    "with open(outfile, 'w') as file:",
+    "    file.write(str(result))",
+    "",
+    "print('done')"
+]
 IDMC_UTILS_MODULE = [
     "import matplotlib.pyplot as plt",
     "from sklearn import mixture",

tests/test_base.py

@@ -8,10 +8,9 @@ from meow_base.core.base_handler import BaseHandler
 from meow_base.core.base_monitor import BaseMonitor
 from meow_base.core.base_pattern import BasePattern
 from meow_base.core.base_recipe import BaseRecipe
-from meow_base.core.base_rule import BaseRule
 from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
 from meow_base.patterns.file_event_pattern import FileEventPattern
-from shared import valid_pattern_one, valid_recipe_one, setup, teardown
+from shared import setup, teardown
 
 
 class BaseRecipeTests(unittest.TestCase):
@@ -147,35 +146,6 @@ class BasePatternTests(unittest.TestCase):
         self.assertEqual(len(values), 0)
 
 
-class BaseRuleTests(unittest.TestCase):
-    def setUp(self)->None:
-        super().setUp()
-        setup()
-
-    def tearDown(self)->None:
-        super().tearDown()
-        teardown()
-
-    # Test that BaseRecipe instantiation
-    def testBaseRule(self)->None:
-        with self.assertRaises(TypeError):
-            BaseRule("name", "", "")
-
-        class NewRule(BaseRule):
-            pass
-        with self.assertRaises(NotImplementedError):
-            NewRule("name", "", "")
-
-        class FullRule(BaseRule):
-            pattern_type = "pattern"
-            recipe_type = "recipe"
-            def _is_valid_recipe(self, recipe:Any)->None:
-                pass
-            def _is_valid_pattern(self, pattern:Any)->None:
-                pass
-        FullRule("name", valid_pattern_one, valid_recipe_one)
-
-
 class BaseMonitorTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()

tests/test_functionality.py

@@ -11,7 +11,7 @@ from sys import prefix, base_prefix
 from time import sleep
 from typing import Dict
 
-from meow_base.core.base_rule import BaseRule
+from meow_base.core.rule import Rule
 from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
     SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
     WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
@@ -581,7 +581,7 @@ class MeowTests(unittest.TestCase):
     def testCreateRule(self)->None:
         rule = create_rule(valid_pattern_one, valid_recipe_one)
 
-        self.assertIsInstance(rule, BaseRule)
+        self.assertIsInstance(rule, Rule)
 
         with self.assertRaises(ValueError):
             rule = create_rule(valid_pattern_one, valid_recipe_two)
@@ -607,7 +607,7 @@ class MeowTests(unittest.TestCase):
         self.assertEqual(len(rules), 2)
         for k, rule in rules.items():
             self.assertIsInstance(k, str)
-            self.assertIsInstance(rule, BaseRule)
+            self.assertIsInstance(rule, Rule)
             self.assertEqual(k, rule.name)
 
     # Test that create_rules creates nothing from invalid pattern inputs

tests/test_recipes.py

@@ -1,6 +1,8 @@
 
 import jsonschema
 import os
+import stat
+import subprocess
 import unittest
 
 from multiprocessing import Pipe
@@ -11,22 +13,27 @@ from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, \
     EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \
     JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
     JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \
-    SWEEP_START, JOB_TYPE_PAPERMILL, get_base_file, get_job_file, \
+    SWEEP_START, JOB_TYPE_PAPERMILL, JOB_TYPE_BASH, \
-    get_result_file
+    get_base_file, get_job_file, get_result_file
+from meow_base.core.rule import Rule
 from meow_base.functionality.file_io import lines_to_string, make_dir, \
     read_yaml, write_file, write_notebook, write_yaml
 from meow_base.functionality.hashing import get_file_hash
 from meow_base.functionality.meow import create_job, create_rules, \
     create_rule, create_watchdog_event
+from meow_base.functionality.parameterisation import parameterize_bash_script
 from meow_base.patterns.file_event_pattern import FileEventPattern
+from meow_base.recipes.bash_recipe import BashRecipe, BashHandler, \
+    assemble_bash_job_script
 from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
     PapermillHandler, papermill_job_func, get_recipe_from_notebook
 from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, \
     python_job_func
-from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule
 from shared import BAREBONES_PYTHON_SCRIPT, COMPLETE_PYTHON_SCRIPT, \
     TEST_JOB_QUEUE, TEST_MONITOR_BASE, TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, \
-    APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, setup, teardown
+    APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, BAREBONES_BASH_SCRIPT, \
+    COMPLETE_BASH_SCRIPT, \
+    setup, teardown
 
 class JupyterNotebookTests(unittest.TestCase):
     def setUp(self)->None:
@@ -157,7 +164,7 @@ class PapermillHandlerTests(unittest.TestCase):
         rules = create_rules(patterns, recipes)
         self.assertEqual(len(rules), 1)
         _, rule = rules.popitem()
-        self.assertIsInstance(rule, FileEventJupyterNotebookRule)
+        self.assertIsInstance(rule, Rule)
 
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
@@ -208,7 +215,7 @@ class PapermillHandlerTests(unittest.TestCase):
         rules = create_rules(patterns, recipes)
         self.assertEqual(len(rules), 1)
         _, rule = rules.popitem()
-        self.assertIsInstance(rule, FileEventJupyterNotebookRule)
+        self.assertIsInstance(rule, Rule)
 
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
@@ -278,7 +285,7 @@ class PapermillHandlerTests(unittest.TestCase):
         rules = create_rules(patterns, recipes)
         self.assertEqual(len(rules), 1)
         _, rule = rules.popitem()
-        self.assertIsInstance(rule, FileEventJupyterNotebookRule)
+        self.assertIsInstance(rule, Rule)
 
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
@@ -576,7 +583,7 @@ class PythonHandlerTests(unittest.TestCase):
         rules = create_rules(patterns, recipes)
         self.assertEqual(len(rules), 1)
         _, rule = rules.popitem()
-        self.assertIsInstance(rule, FileEventPythonRule)
+        self.assertIsInstance(rule, Rule)
 
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
@@ -627,7 +634,7 @@ class PythonHandlerTests(unittest.TestCase):
         rules = create_rules(patterns, recipes)
         self.assertEqual(len(rules), 1)
         _, rule = rules.popitem()
-        self.assertIsInstance(rule, FileEventPythonRule)
+        self.assertIsInstance(rule, Rule)
 
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
@@ -697,7 +704,7 @@ class PythonHandlerTests(unittest.TestCase):
         rules = create_rules(patterns, recipes)
         self.assertEqual(len(rules), 1)
         _, rule = rules.popitem()
-        self.assertIsInstance(rule, FileEventPythonRule)
+        self.assertIsInstance(rule, Rule)
 
         self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
 
@@ -830,6 +837,12 @@ class PythonHandlerTests(unittest.TestCase):
 
         self.assertTrue(os.path.exists(result_path))
 
+        with open(result_path, "r") as f:
+            result = f.read()
+
+        self.assertEqual(result, "124937.5")
+
+
     # Test jobFunc doesn't execute with no args
     def testJobFuncBadArgs(self)->None:
         try:
@@ -878,3 +891,425 @@ class PythonHandlerTests(unittest.TestCase):
             EVENT_RULE: rule
         })
         self.assertTrue(status)
+
+class BashTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+ 
+    # Test BashRecipe can be created
+    def testBashRecipeCreationMinimum(self)->None:
+        BashRecipe("test_recipe", BAREBONES_BASH_SCRIPT)
+
+    # Test BashRecipe cannot be created without name
+    def testBashRecipeCreationNoName(self)->None:
+        with self.assertRaises(ValueError):
+            BashRecipe("", BAREBONES_BASH_SCRIPT)
+
+    # Test BashRecipe cannot be created with invalid name
+    def testBashRecipeCreationInvalidName(self)->None:
+        with self.assertRaises(ValueError):
+            BashRecipe("@test_recipe", BAREBONES_BASH_SCRIPT)
+
+    # Test BashRecipe cannot be created with invalid recipe
+    def testBashRecipeCreationInvalidRecipe(self)->None:
+        with self.assertRaises(TypeError):
+            BashRecipe("test_recipe", BAREBONES_NOTEBOOK)
+
+    # Test BashRecipe name setup correctly
+    def testBashRecipeSetupName(self)->None:
+        name = "name"
+        pr = BashRecipe(name, BAREBONES_BASH_SCRIPT)
+        self.assertEqual(pr.name, name)
+
+    # Test BashRecipe recipe setup correctly
+    def testBashRecipeSetupRecipe(self)->None:
+        pr = BashRecipe("name", BAREBONES_BASH_SCRIPT)
+        self.assertEqual(pr.recipe, BAREBONES_BASH_SCRIPT)
+
+    # Test BashRecipe parameters setup correctly
+    def testBashRecipeSetupParameters(self)->None:
+        parameters = {
+            "a": 1,
+            "b": True
+        }
+        pr = BashRecipe(
+            "name", BAREBONES_BASH_SCRIPT, parameters=parameters)
+        self.assertEqual(pr.parameters, parameters)
+
+    # Test BashRecipe requirements setup correctly
+    def testBashRecipeSetupRequirements(self)->None:
+        requirements = {
+            "a": 1,
+            "b": True
+        }
+        pr = BashRecipe(
+            "name", BAREBONES_BASH_SCRIPT, requirements=requirements)
+        self.assertEqual(pr.requirements, requirements)
+
+class BashHandlerTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+
+    # Test BashHandler can be created
+    def testBashHandlerMinimum(self)->None:
+        BashHandler(job_queue_dir=TEST_JOB_QUEUE)
+
+    # Test BashHandler naming
+    def testBashHandlerNaming(self)->None:
+        test_name = "test_name"
+        handler = BashHandler(name=test_name)
+        self.assertEqual(handler.name, test_name)
+
+        handler = BashHandler()
+        self.assertTrue(handler.name.startswith("handler_"))
+
+    # Test BashHandler will handle given events
+    def testBashHandlerHandling(self)->None:
+        from_handler_reader, from_handler_writer = Pipe()
+        ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
+        ph.to_runner = from_handler_writer
+        
+        with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
+            f.write("Data")
+
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one")
+        recipe = BashRecipe(
+            "recipe_one", COMPLETE_BASH_SCRIPT)
+
+        patterns = {
+            pattern_one.name: pattern_one,
+        }
+        recipes = {
+            recipe.name: recipe,
+        }
+
+        rules = create_rules(patterns, recipes)
+        self.assertEqual(len(rules), 1)
+        _, rule = rules.popitem()
+        self.assertIsInstance(rule, Rule)
+
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+        event = {
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
+            WATCHDOG_BASE: TEST_MONITOR_BASE,
+            EVENT_RULE: rule,
+            WATCHDOG_HASH: get_file_hash(
+                os.path.join(TEST_MONITOR_BASE, "A"), SHA256
+            )
+        }
+
+        ph.handle(event)
+
+        if from_handler_reader.poll(3):
+            job_dir = from_handler_reader.recv()
+
+        self.assertIsInstance(job_dir, str)
+        self.assertTrue(os.path.exists(job_dir))
+
+        job = read_yaml(os.path.join(job_dir, META_FILE))
+        valid_job(job)
+
+    # Test BashHandler will create enough jobs from single sweep
+    def testBashHandlerHandlingSingleSweep(self)->None:
+        from_handler_reader, from_handler_writer = Pipe()
+        ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
+        ph.to_runner = from_handler_writer
+        
+        with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
+            f.write("Data")
+
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one", sweep={"s":{
+                SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
+            }})
+        recipe = BashRecipe(
+            "recipe_one", COMPLETE_BASH_SCRIPT)
+
+        patterns = {
+            pattern_one.name: pattern_one,
+        }
+        recipes = {
+            recipe.name: recipe,
+        }
+
+        rules = create_rules(patterns, recipes)
+        self.assertEqual(len(rules), 1)
+        _, rule = rules.popitem()
+        self.assertIsInstance(rule, Rule)
+
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+        event = {
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
+            WATCHDOG_BASE: TEST_MONITOR_BASE,
+            EVENT_RULE: rule,
+            WATCHDOG_HASH: get_file_hash(
+                os.path.join(TEST_MONITOR_BASE, "A"), SHA256
+            )
+        }
+
+        ph.handle(event)
+
+        jobs = []
+        recieving = True
+        while recieving:
+            if from_handler_reader.poll(3):
+                jobs.append(from_handler_reader.recv())
+            else:
+                recieving = False
+
+        values = [0, 1, 2]
+        self.assertEqual(len(jobs), 3)
+        for job_dir in jobs:
+            self.assertIsInstance(job_dir, str)
+            self.assertTrue(os.path.exists(job_dir))
+
+            job = read_yaml(os.path.join(job_dir, META_FILE))
+            valid_job(job)
+
+            self.assertIn(JOB_PARAMETERS, job)
+            self.assertIn("s", job[JOB_PARAMETERS])
+            if job[JOB_PARAMETERS]["s"] in values:
+                values.remove(job[JOB_PARAMETERS]["s"])
+        self.assertEqual(len(values), 0)
+
+    # Test BashHandler will create enough jobs from multiple sweeps
+    def testBashHandlerHandlingMultipleSweep(self)->None:
+        from_handler_reader, from_handler_writer = Pipe()
+        ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
+        ph.to_runner = from_handler_writer
+        
+        with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
+            f.write("Data")
+
+        pattern_one = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one", sweep={
+                "s1":{
+                    SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
+                },
+                "s2":{
+                    SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
+                }
+            })
+        recipe = BashRecipe(
+            "recipe_one", COMPLETE_BASH_SCRIPT)
+
+        patterns = {
+            pattern_one.name: pattern_one,
+        }
+        recipes = {
+            recipe.name: recipe,
+        }
+
+        rules = create_rules(patterns, recipes)
+        self.assertEqual(len(rules), 1)
+        _, rule = rules.popitem()
+        self.assertIsInstance(rule, Rule)
+
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+        event = {
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
+            WATCHDOG_BASE: TEST_MONITOR_BASE,
+            EVENT_RULE: rule,
+            WATCHDOG_HASH: get_file_hash(
+                os.path.join(TEST_MONITOR_BASE, "A"), SHA256
+            )
+        }
+
+        ph.handle(event)
+
+        jobs = []
+        recieving = True
+        while recieving:
+            if from_handler_reader.poll(3):
+                jobs.append(from_handler_reader.recv())
+            else:
+                recieving = False
+
+        values = [
+            "s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20", 
+            "s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35", 
+            "s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50", 
+            "s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65", 
+            "s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80", 
+        ]
+        self.assertEqual(len(jobs), 15)
+        for job_dir in jobs:
+            self.assertIsInstance(job_dir, str)
+            self.assertTrue(os.path.exists(job_dir))
+
+            job = read_yaml(os.path.join(job_dir, META_FILE))
+            valid_job(job)
+
+            self.assertIn(JOB_PARAMETERS, job)
+            val1 = None
+            val2 = None
+            if "s1" in job[JOB_PARAMETERS]:
+                val1 = f"s1-{job[JOB_PARAMETERS]['s1']}"
+            if "s2" in job[JOB_PARAMETERS]:
+                val2 = f"s2-{job[JOB_PARAMETERS]['s2']}"
+            val = None
+            if val1 and val2:
+                val = f"{val1}/{val2}"
+            if val and val in values:
+                values.remove(val)
+        self.assertEqual(len(values), 0)
+
+    # Test jobFunc performs as expected
+    def testJobFunc(self)->None:
+        file_path = os.path.join(TEST_MONITOR_BASE, "test")
+        result_path = os.path.join(TEST_MONITOR_BASE, "output")
+
+        with open(file_path, "w") as f:
+            f.write("250")
+
+        file_hash = get_file_hash(file_path, SHA256)
+
+        pattern = FileEventPattern(
+            "pattern", 
+            file_path, 
+            "recipe_one", 
+            "infile", 
+            parameters={
+                "extra":"A line from a test Pattern",
+                "outfile": result_path
+            })
+        recipe = BashRecipe(
+            "recipe_one", COMPLETE_BASH_SCRIPT)
+
+        rule = create_rule(pattern, recipe)
+
+        params_dict = {
+            "extra":"extra",
+            "infile":file_path,
+            "outfile": result_path
+        }
+
+        job_dict = create_job(
+            JOB_TYPE_BASH,
+            create_watchdog_event(
+                file_path,
+                rule,
+                TEST_MONITOR_BASE,
+                file_hash
+            ),
+            extras={
+                JOB_PARAMETERS:params_dict,
+                JOB_HASH: file_hash
+            }
+        )
+
+        job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
+        make_dir(job_dir)
+
+        meta_file = os.path.join(job_dir, META_FILE)
+        write_yaml(job_dict, meta_file)
+
+        base_script = parameterize_bash_script(
+            COMPLETE_BASH_SCRIPT, params_dict
+        )
+        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))
+        write_file(lines_to_string(base_script), base_file)
+        st = os.stat(base_file)
+        os.chmod(base_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+        job_script = assemble_bash_job_script()
+        job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))
+        write_file(lines_to_string(job_script), job_file)
+        st = os.stat(job_file)
+        os.chmod(job_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+
+        print(os.listdir(job_dir))
+        print(os.getcwd())
+
+        result = subprocess.call(job_file, cwd=".")
+
+        self.assertEqual(result, 0)
+
+        self.assertTrue(os.path.exists(job_dir))
+        meta_path = os.path.join(job_dir, META_FILE)
+        self.assertTrue(os.path.exists(meta_path))
+
+        status = read_yaml(meta_path)
+        self.assertIsInstance(status, Dict)
+        self.assertIn(JOB_STATUS, status)
+        self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])    
+        self.assertNotIn(JOB_ERROR, status)
+
+        self.assertTrue(os.path.exists(
+            os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))))
+        self.assertTrue(os.path.exists(
+            os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))))
+
+        self.assertTrue(os.path.exists(result_path))
+
+        with open(result_path, "r") as f:
+            result = f.read()
+
+        self.assertEqual(result, "124937\n")
+
+    # Test jobFunc doesn't execute with no args
+    def testJobFuncBadArgs(self)->None:
+        try:
+            Bash_job_func({})
+        except Exception:
+            pass
+
+        self.assertEqual(len(os.listdir(TEST_JOB_QUEUE)), 0)
+        self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
+
+    # Test handling criteria function
+    def testValidHandleCriteria(self)->None:
+        ph = BashHandler()
+
+        pattern = FileEventPattern(
+            "pattern_one", "A", "recipe_one", "file_one")
+        recipe = BashRecipe(
+            "recipe_one", BAREBONES_BASH_SCRIPT
+        )
+
+        rule = create_rule(pattern, recipe)
+
+        status, _ = ph.valid_handle_criteria({})
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria("")
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: "type",
+            EVENT_RULE: rule
+        })
+        self.assertFalse(status)
+
+        status, _ = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_RULE: "rule"
+        })
+        self.assertFalse(status)
+
+        status, s = ph.valid_handle_criteria({
+            EVENT_PATH: "path",
+            EVENT_TYPE: EVENT_TYPE_WATCHDOG,
+            EVENT_RULE: rule
+        })
+        self.assertTrue(status)

tests/test_rule.py

@@ -0,0 +1,92 @@
+
+import unittest
+
+from meow_base.core.rule import Rule
+from meow_base.patterns.file_event_pattern import FileEventPattern
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
+from shared import BAREBONES_NOTEBOOK, setup, teardown
+
+class CorrectnessTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+        
+    # Test Rule created from valid pattern and recipe
+    def testRuleCreationMinimum(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        Rule(fep, jnr)
+
+    # Test Rule not created with empty name
+    def testRuleCreationNoName(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        r = Rule(fep, jnr)
+
+        self.assertIsInstance(r.name, str)
+        self.assertTrue(len(r.name) > 1)
+
+    # Test Rule not created with invalid name
+    def testRuleCreationInvalidName(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        with self.assertRaises(TypeError):
+            Rule(fep, jnr, name=1)
+
+    # Test Rule not created with invalid pattern
+    def testRuleCreationInvalidPattern(self)->None:
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        with self.assertRaises(TypeError):
+            Rule("pattern", jnr)
+
+    # Test Rule not created with invalid recipe
+    def testRuleCreationInvalidRecipe(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+
+        with self.assertRaises(TypeError):
+            Rule(fep, "recipe")
+
+    # Test Rule not created with mismatched recipe
+    def testRuleCreationMissmatchedRecipe(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
+
+        with self.assertRaises(ValueError):
+            Rule(fep, jnr)
+
+    # Test Rule created with valid name
+    def testRuleSetupName(self)->None:
+        name = "name"
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        fejnr = Rule(fep, jnr, name=name)
+
+        self.assertEqual(fejnr.name, name)
+
+    # Test Rule not created with valid pattern
+    def testRuleSetupPattern(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        fejnr = Rule(fep, jnr)
+
+        self.assertEqual(fejnr.pattern, fep)
+
+    # Test Rule not created with valid recipe
+    def testRuleSetupRecipe(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        fejnr = Rule(fep, jnr)
+
+        self.assertEqual(fejnr.recipe, jnr)
+

tests/test_rules.py

@@ -1,91 +0,0 @@
-
-import unittest
-
-from meow_base.patterns.file_event_pattern import FileEventPattern
-from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
-from meow_base.rules.file_event_jupyter_notebook_rule import \
-    FileEventJupyterNotebookRule
-from shared import BAREBONES_NOTEBOOK, setup, teardown
-
-class CorrectnessTests(unittest.TestCase):
-    def setUp(self)->None:
-        super().setUp()
-        setup()
-
-    def tearDown(self)->None:
-        super().tearDown()
-        teardown()
-        
-    # Test FileEventJupyterNotebookRule created from valid pattern and recipe
-    def testFileEventJupyterNotebookRuleCreationMinimum(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        FileEventJupyterNotebookRule("name", fep, jnr)
-
-    # Test FileEventJupyterNotebookRule not created with empty name
-    def testFileEventJupyterNotebookRuleCreationNoName(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(ValueError):
-            FileEventJupyterNotebookRule("", fep, jnr)
-
-    # Test FileEventJupyterNotebookRule not created with invalid name
-    def testFileEventJupyterNotebookRuleCreationInvalidName(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(TypeError):
-            FileEventJupyterNotebookRule(1, fep, jnr)
-
-    # Test FileEventJupyterNotebookRule not created with invalid pattern
-    def testFileEventJupyterNotebookRuleCreationInvalidPattern(self)->None:
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(TypeError):
-            FileEventJupyterNotebookRule("name", "pattern", jnr)
-
-    # Test FileEventJupyterNotebookRule not created with invalid recipe
-    def testFileEventJupyterNotebookRuleCreationInvalidRecipe(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-
-        with self.assertRaises(TypeError):
-            FileEventJupyterNotebookRule("name", fep, "recipe")
-
-    # Test FileEventJupyterNotebookRule not created with mismatched recipe
-    def testFileEventJupyterNotebookRuleCreationMissmatchedRecipe(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(ValueError):
-            FileEventJupyterNotebookRule("name", fep, jnr)
-
-    # Test FileEventJupyterNotebookRule created with valid name
-    def testFileEventJupyterNotebookRuleSetupName(self)->None:
-        name = "name"
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        fejnr = FileEventJupyterNotebookRule(name, fep, jnr)
-
-        self.assertEqual(fejnr.name, name)
-
-    # Test FileEventJupyterNotebookRule not created with valid pattern
-    def testFileEventJupyterNotebookRuleSetupPattern(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        fejnr = FileEventJupyterNotebookRule("name", fep, jnr)
-
-        self.assertEqual(fejnr.pattern, fep)
-
-    # Test FileEventJupyterNotebookRule not created with valid recipe
-    def testFileEventJupyterNotebookRuleSetupRecipe(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        fejnr = FileEventJupyterNotebookRule("name", fep, jnr)
-
-        self.assertEqual(fejnr.recipe, jnr)
-