✨ Small change to performance test

.gitignore

@@ -58,6 +58,10 @@ tests/test_data
 tests/job_output
 tests/job_queue
 tests/Backup*
+benchmarking/benchmark_base
+benchmarking/job_output
+benchmarking/job_queue
+benchmarking/result*
 
 # hdf5
 *.h5

README.md

@@ -12,7 +12,12 @@ The most import definitions are the BasePattern, and BaseRecipe, which define th
 The way to run a MEOW system is to create and MeowRunner instance, found in **core/runner.py**. This will take 1 or more Monitors, 1 or more Handlers, and 1 or more Conductors. In turn, these will listen for events, respond to events, and execute any analysis identified. Examples of how this can be run can be found in **tests/testRunner.py**
 
 ## Testing
-Pytest unittests are provided within the 'tests directory, as well as a script **test_all.sh** for calling all test scripts from a single command. Individual test scripts can be started using:
+Pytest unittests are provided within the 'tests directory, as well as a script **test_all.sh** for calling all test scripts from a single command. This can be
+run as:
+
+    test_all.sh -s
+
+This will skip the more time consuming tests if you just need quick feedback. Without the -s argument then all tests will be run. Individual test scripts can be started using:
 
     pytest test_runner.py::MeowTests -W ignore::DeprecationWarning
 
@@ -24,3 +29,6 @@ to run locally, update your '~/.bashrc' file to inclue:
 
     # Manually added to get local testing of Python files working easier
     export PYTHONPATH=/home/patch/Documents/Research/Python
+
+## Benchmarking 
+Benchmarks have been written for this library, but are included in the seperate https://github.com/PatchOfScotland/meow_benchmarks package

conductors/__init__.py

@@ -1,2 +1,3 @@
 
-from conductors.local_python_conductor import LocalPythonConductor
+from .local_python_conductor import LocalPythonConductor
+from .local_bash_conductor import LocalBashConductor

conductors/local_bash_conductor.py

@@ -0,0 +1,63 @@
+
+"""
+This file contains definitions for the LocalBashConductor, in order to 
+execute Bash jobs on the local resource.
+
+Author(s): David Marchant
+"""
+
+import os
+
+from typing import Any, Dict, Tuple
+
+from meow_base.core.base_conductor import BaseConductor
+from meow_base.core.meow import valid_job
+from meow_base.core.vars import DEFAULT_JOB_QUEUE_DIR, \
+    DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, JOB_TYPE, \
+    DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
+from meow_base.functionality.validation import valid_dir_path
+from meow_base.functionality.file_io import make_dir
+
+
+class LocalBashConductor(BaseConductor):
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
+            job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="", 
+            pause_time:int=5)->None:
+        """LocalBashConductor Constructor. This should be used to execute 
+        Bash jobs, and will then pass any internal job runner files to the 
+        output directory. Note that if this handler is given to a MeowRunner
+        object, the job_queue_dir and job_output_dir will be overwridden."""
+        super().__init__(name=name, pause_time=pause_time)
+        self._is_valid_job_queue_dir(job_queue_dir)
+        self.job_queue_dir = job_queue_dir
+        self._is_valid_job_output_dir(job_output_dir)
+        self.job_output_dir = job_output_dir
+
+    def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
+        """Function to determine given an job defintion, if this conductor can 
+        process it or not. This conductor will accept any Bash job type"""
+        try:
+            valid_job(job)
+            msg = ""
+            if job[JOB_TYPE] not in [JOB_TYPE_BASH]:
+                msg = f"Job type was not {JOB_TYPE_BASH}."
+            if msg:
+                return False, msg
+            else:
+                return True, ""
+        except Exception as e:
+            return False, str(e)
+
+    def _is_valid_job_queue_dir(self, job_queue_dir)->None:
+        """Validation check for 'job_queue_dir' variable from main 
+        constructor."""
+        valid_dir_path(job_queue_dir, must_exist=False)
+        if not os.path.exists(job_queue_dir):
+            make_dir(job_queue_dir)
+
+    def _is_valid_job_output_dir(self, job_output_dir)->None:
+        """Validation check for 'job_output_dir' variable from main 
+        constructor."""
+        valid_dir_path(job_output_dir, must_exist=False)
+        if not os.path.exists(job_output_dir):
+            make_dir(job_output_dir)

conductors/local_python_conductor.py

@@ -11,23 +11,26 @@ import shutil
 from datetime import datetime
 from typing import Any, Tuple, Dict
 
-from core.base_conductor import BaseConductor
+from meow_base.core.base_conductor import BaseConductor
-from core.correctness.meow import valid_job
+from meow_base.core.meow import valid_job
-from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
+from meow_base.core.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, \
-    STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
+    JOB_STATUS, STATUS_RUNNING, JOB_START_TIME, META_FILE, \
-    STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
+    BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
-    JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
+    JOB_ERROR, JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, \
-from core.correctness.validation import valid_dir_path
+    DEFAULT_JOB_OUTPUT_DIR
-from functionality.file_io import make_dir, read_yaml, write_file, write_yaml
+from meow_base.functionality.validation import valid_dir_path
+from meow_base.functionality.file_io import make_dir, write_file, \
+    threadsafe_read_status, threadsafe_update_status
 
 class LocalPythonConductor(BaseConductor):
     def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
-            job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR)->None:
+            job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="", 
+            pause_time:int=5)->None:
         """LocalPythonConductor Constructor. This should be used to execute 
         Python jobs, and will then pass any internal job runner files to the 
         output directory. Note that if this handler is given to a MeowRunner
         object, the job_queue_dir and job_output_dir will be overwridden."""
-        super().__init__()
+        super().__init__(name=name, pause_time=pause_time)
         self._is_valid_job_queue_dir(job_queue_dir)
         self.job_queue_dir = job_queue_dir
         self._is_valid_job_output_dir(job_output_dir)
@@ -48,68 +51,6 @@ class LocalPythonConductor(BaseConductor):
         except Exception as e:
             return False, str(e)
 
-    def execute(self, job_dir:str)->None:
-        """Function to actually execute a Python job. This will read job 
-        defintions from its meta file, update the meta file and attempt to 
-        execute. Some unspecific feedback will be given on execution failure, 
-        but depending on what it is it may be up to the job itself to provide 
-        more detailed feedback."""
-        valid_dir_path(job_dir, must_exist=True)
-
-        # Test our job parameters. Even if its gibberish, we still move to 
-        # output
-        abort = False
-        try:
-            meta_file = os.path.join(job_dir, META_FILE)
-            job = read_yaml(meta_file)
-            valid_job(job)
-
-            # update the status file with running status
-            job[JOB_STATUS] = STATUS_RUNNING
-            job[JOB_START_TIME] = datetime.now()
-            write_yaml(job, meta_file)
-
-        except Exception as e:
-            # If something has gone wrong at this stage then its bad, so we 
-            # need to make our own error file
-            error_file = os.path.join(job_dir, BACKUP_JOB_ERROR_FILE)
-            write_file(f"Recieved incorrectly setup job.\n\n{e}", error_file)
-            abort = True
-
-        # execute the job
-        if not abort:
-            try:
-                job_function = job[PYTHON_FUNC]
-                job_function(job_dir)
-
-                # get up to date job data
-                job = read_yaml(meta_file)
-
-                # Update the status file with the finalised status
-                job[JOB_STATUS] = STATUS_DONE
-                job[JOB_END_TIME] = datetime.now()
-                write_yaml(job, meta_file)
-
-            except Exception as e:
-                # get up to date job data
-                job = read_yaml(meta_file)
-
-                # Update the status file with the error status. Don't overwrite
-                # any more specific error messages already created
-                if JOB_STATUS not in job:
-                    job[JOB_STATUS] = STATUS_FAILED
-                if JOB_END_TIME not in job:
-                    job[JOB_END_TIME] = datetime.now()
-                if JOB_ERROR not in job:
-                    job[JOB_ERROR] = f"Job execution failed. {e}"
-                write_yaml(job, meta_file)
-
-        # Move the contents of the execution directory to the final output 
-        # directory. 
-        job_output_dir = \
-            os.path.join(self.job_output_dir, os.path.basename(job_dir))
-        shutil.move(job_dir, job_output_dir)
-
     def _is_valid_job_queue_dir(self, job_queue_dir)->None:
         """Validation check for 'job_queue_dir' variable from main 
         constructor."""

core/base_conductor.py

@@ -5,14 +5,37 @@ from for all conductor instances.
 
 Author(s): David Marchant
 """
+import shutil
+import subprocess
+import os 
 
-from typing import Any, Tuple, Dict
+from datetime import datetime
+from threading import Event, Thread
+from time import sleep
+from typing import Any, Tuple, Dict, Union
 
-from core.correctness.vars import get_drt_imp_msg
+
-from core.correctness.validation import check_implementation
+from meow_base.core.meow import valid_job
+from meow_base.core.vars import VALID_CONDUCTOR_NAME_CHARS, VALID_CHANNELS, \
+    JOB_STATUS, JOB_START_TIME, META_FILE, STATUS_RUNNING, STATUS_DONE , \
+    BACKUP_JOB_ERROR_FILE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
+    get_drt_imp_msg
+from meow_base.functionality.file_io import write_file, \
+    threadsafe_read_status, threadsafe_update_status
+from meow_base.functionality.validation import check_implementation, \
+    valid_string, valid_existing_dir_path, valid_natural, valid_dir_path
+from meow_base.functionality.naming import generate_conductor_id
 
 
 class BaseConductor:
+    # An identifier for a conductor within the runner. Can be manually set in 
+    # the constructor, or autogenerated if no name provided.
+    name:str
+    # A channel for sending messages to the runner job queue. Note that this 
+    # will be overridden by a MeowRunner, if a conductor instance is passed to 
+    # it, and so does not need to be initialised within the conductor itself, 
+    # unless the conductor is running independently of a runner.
+    to_runner_job: VALID_CHANNELS
     # Directory where queued jobs are initially written to. Note that this 
     # will be overridden by a MeowRunner, if a handler instance is passed to 
     # it, and so does not need to be initialised within the handler itself.
@@ -21,11 +44,19 @@ class BaseConductor:
     # will be overridden by a MeowRunner, if a handler instance is passed to 
     # it, and so does not need to be initialised within the handler itself.
     job_output_dir:str
-    def __init__(self)->None:
+    # A count, for how long a conductor will wait if told that there are no 
+    # jobs in the runner, before polling again. Default is 5 seconds.
+    pause_time: int
+    def __init__(self, name:str="", pause_time:int=5)->None:
         """BaseConductor Constructor. This will check that any class inheriting
         from it implements its validation functions."""
-        check_implementation(type(self).execute, BaseConductor)
         check_implementation(type(self).valid_execute_criteria, BaseConductor)
+        if not name:
+            name = generate_conductor_id()
+        self._is_valid_name(name)
+        self.name = name    
+        self._is_valid_pause_time(pause_time)
+        self.pause_time = pause_time
 
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 
@@ -35,12 +66,165 @@ class BaseConductor:
             raise TypeError(msg)
         return object.__new__(cls)
 
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_CONDUCTOR_NAME_CHARS)
+
+    def _is_valid_pause_time(self, pause_time:int)->None:
+        """Validation check for 'pause_time' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_natural(pause_time, hint="BaseHandler.pause_time")
+
+    def prompt_runner_for_job(self)->Union[Dict[str,Any],Any]:
+        self.to_runner_job.send(1)
+
+        if self.to_runner_job.poll(self.pause_time):
+            return self.to_runner_job.recv()
+        return None
+
+    def start(self)->None:
+        """Function to start the conductor as an ongoing thread, as defined by 
+        the main_loop function. Together, these will execute any code in a 
+        implemented conductors execute function sequentially, but concurrently 
+        to any other conductors running or other runner operations. This is 
+        intended as a naive mmultiprocessing implementation, and any more in 
+        depth parallelisation of execution must be implemented by a user by 
+        overriding this function, and the stop function."""
+        self._stop_event = Event()        
+        self._handle_thread = Thread(
+            target=self.main_loop, 
+            args=(self._stop_event,),
+            daemon=True,
+            name="conductor_thread"
+        )
+        self._handle_thread.start()
+
+    def stop(self)->None:
+        """Function to stop the conductor as an ongoing thread. May be 
+        overidden by any child class. This function should also be overriden if
+        the start function has been."""
+        self._stop_event.set()
+        self._handle_thread.join()
+        
+    def main_loop(self, stop_event)->None:
+        """Function defining an ongoing thread, as started by the start 
+        function and stoped by the stop function. """
+
+        while not stop_event.is_set():
+            reply = self.prompt_runner_for_job()
+
+            # If we have recieved 'None' then we have already timed out so skip 
+            # this loop and start again
+            if reply is None:
+                continue
+
+            try:
+                valid_existing_dir_path(reply)
+            except:
+                # Were not given a job dir, so sleep before trying again
+                sleep(self.pause_time)
+            
+            try:
+                self.execute(reply)
+            except:
+                # TODO some error reporting here
+                pass
+ 
     def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
         """Function to determine given an job defintion, if this conductor can 
         process it or not. Must be implemented by any child process."""
         pass
 
+    def run_job(self, job_dir:str)->None:
+        """Function to actually execute a job. This will read job 
+        defintions from its meta file, update the meta file and attempt to 
+        execute. Some unspecific feedback will be given on execution failure, 
+        but depending on what it is it may be up to the job itself to provide 
+        more detailed feedback. If you simply wish to alter the conditions 
+        under which the job is executed, please instead look at the execute 
+        function."""
+        valid_dir_path(job_dir, must_exist=True)
+
+        # Test our job parameters. Even if its gibberish, we still move to 
+        # output
+        abort = False
+        try:
+            meta_file = os.path.join(job_dir, META_FILE)
+            job = threadsafe_read_status(meta_file)
+            valid_job(job)
+
+            # update the status file with running status
+            threadsafe_update_status(
+                {
+                    JOB_STATUS: STATUS_RUNNING,
+                    JOB_START_TIME: datetime.now()
+                }, 
+                meta_file
+            )
+
+        except Exception as e:
+            # If something has gone wrong at this stage then its bad, so we 
+            # need to make our own error file
+            error_file = os.path.join(job_dir, BACKUP_JOB_ERROR_FILE)
+            write_file(f"Recieved incorrectly setup job.\n\n{e}", error_file)
+            abort = True
+
+        # execute the job
+        if not abort:
+            try:
+                result = subprocess.call(
+                    os.path.join(job_dir, job["tmp script command"]), 
+                    cwd="."
+                )
+
+                if result == 0:
+                    # Update the status file with the finalised status
+                    threadsafe_update_status(
+                        {
+                            JOB_STATUS: STATUS_DONE,
+                            JOB_END_TIME: datetime.now()
+                        }, 
+                        meta_file
+                    )
+
+                else:
+                    # Update the status file with the error status. Don't 
+                    # overwrite any more specific error messages already 
+                    # created
+                    threadsafe_update_status(
+                        {
+                            JOB_STATUS: STATUS_FAILED,
+                            JOB_END_TIME: datetime.now(),
+                            JOB_ERROR: "Job execution returned non-zero."
+                        },
+                        meta_file
+                    )
+
+            except Exception as e:
+                # Update the status file with the error status. Don't overwrite
+                # any more specific error messages already created
+                threadsafe_update_status(
+                    {
+                        JOB_STATUS: STATUS_FAILED,
+                        JOB_END_TIME: datetime.now(),
+                        JOB_ERROR: f"Job execution failed. {e}"
+                    },
+                    meta_file
+                )
+
+        # Move the contents of the execution directory to the final output 
+        # directory. 
+        job_output_dir = \
+            os.path.join(self.job_output_dir, os.path.basename(job_dir))
+        shutil.move(job_dir, job_output_dir)
+
     def execute(self, job_dir:str)->None:
-        """Function to execute a given job directory. Must be implemented by 
+        """Function to run job execution. By default this will simply call the 
-        any child process."""
+        run_job function, to execute the job locally. However, this function 
-        pass
+        may be overridden to execute the job in some other manner, such as on 
+        another resource. Note that the job itself should be executed using the 
+        run_job func in order to maintain expected logging etc."""
+        self.run_job(job_dir)

core/base_handler.py

@@ -6,26 +6,59 @@ from for all handler instances.
 Author(s): David Marchant
 """
 
-from typing import Any, Tuple, Dict
+import os
+import stat
 
-from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
+from threading import Event, Thread
-from core.correctness.validation import check_implementation
+from typing import Any, Tuple, Dict, Union
+from time import sleep
 
+from meow_base.core.vars import VALID_CHANNELS, EVENT_RULE, EVENT_PATH, \
+    VALID_HANDLER_NAME_CHARS, META_FILE, JOB_ID, JOB_FILE, JOB_PARAMETERS, \
+    get_drt_imp_msg
+from meow_base.core.meow import valid_event
+from meow_base.patterns.file_event_pattern import WATCHDOG_HASH
+from meow_base.functionality.file_io import threadsafe_write_status, \
+    threadsafe_update_status, make_dir, write_file, lines_to_string
+from meow_base.functionality.validation import check_implementation, \
+    valid_string, valid_natural
+from meow_base.functionality.meow import create_job_metadata_dict, \
+    replace_keywords
+from meow_base.functionality.naming import generate_handler_id
 
 class BaseHandler:
-    # A channel for sending messages to the runner. Note that this will be 
+    # An identifier for a handler within the runner. Can be manually set in
-    # overridden by a MeowRunner, if a handler instance is passed to it, and so
+    # the constructor, or autogenerated if no name provided.
-    # does not need to be initialised within the handler itself.
+    name:str
-    to_runner: VALID_CHANNELS
+    # A channel for sending messages to the runner event queue. Note that this
+    # will be overridden by a MeowRunner, if a handler instance is passed to
+    # it, and so does not need to be initialised within the handler itself,
+    # unless the handler is running independently of a runner.
+    to_runner_event: VALID_CHANNELS
+    # A channel for sending messages to the runner job queue. Note that this
+    # will be overridden by a MeowRunner, if a handler instance is passed to
+    # it, and so does not need to be initialised within the handler itself,
+    # unless the handler is running independently of a runner.
+    to_runner_job: VALID_CHANNELS
     # Directory where queued jobs are initially written to. Note that this
     # will be overridden by a MeowRunner, if a handler instance is passed to
     # it, and so does not need to be initialised within the handler itself.
     job_queue_dir:str
-    def __init__(self)->None:
+    # A count, for how long a handler will wait if told that there are no
+    # events in the runner, before polling again. Default is 5 seconds.
+    pause_time: int
+    def __init__(self, name:str='', pause_time:int=5)->None:
         """BaseHandler Constructor. This will check that any class inheriting
         from it implements its validation functions."""
-        check_implementation(type(self).handle, BaseHandler)
         check_implementation(type(self).valid_handle_criteria, BaseHandler)
+        check_implementation(type(self).get_created_job_type, BaseHandler)
+        check_implementation(type(self).create_job_recipe_file, BaseHandler)
+        if not name:
+            name = generate_handler_id()
+        self._is_valid_name(name)
+        self.name = name
+        self._is_valid_pause_time(pause_time)
+        self.pause_time = pause_time
 
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only
@@ -35,12 +68,204 @@ class BaseHandler:
             raise TypeError(msg)
         return object.__new__(cls)
 
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is
+        automatically called during initialisation. This does not need to be
+        overridden by child classes."""
+        valid_string(name, VALID_HANDLER_NAME_CHARS)
+
+    def _is_valid_pause_time(self, pause_time:int)->None:
+        """Validation check for 'pause_time' variable from main constructor. Is
+        automatically called during initialisation. This does not need to be
+        overridden by child classes."""
+        valid_natural(pause_time, hint="BaseHandler.pause_time")
+
+    def prompt_runner_for_event(self)->Union[Dict[str,Any],Any]:
+        self.to_runner_event.send(1)
+
+        if self.to_runner_event.poll(self.pause_time):
+            return self.to_runner_event.recv()
+        return None
+
+    def send_job_to_runner(self, job_id:str)->None:
+        self.to_runner_job.send(job_id)
+
+    def start(self)->None:
+        """Function to start the handler as an ongoing thread, as defined by
+        the main_loop function. Together, these will execute any code in a
+        implemented handlers handle function sequentially, but concurrently to
+        any other handlers running or other runner operations. This is intended
+        as a naive mmultiprocessing implementation, and any more in depth
+        parallelisation of execution must be implemented by a user by
+        overriding this function, and the stop function."""
+        self._stop_event = Event()
+        self._handle_thread = Thread(
+            target=self.main_loop,
+            args=(self._stop_event,),
+            daemon=True,
+            name="handler_thread"
+        )
+        self._handle_thread.start()
+
+    def stop(self)->None:
+        """Function to stop the handler as an ongoing thread. May be overidden
+        by any child class. This function should also be overriden if the start
+        function has been."""
+
+        self._stop_event.set()
+        self._handle_thread.join()
+
+    def main_loop(self, stop_event)->None:
+        """Function defining an ongoing thread, as started by the start
+        function and stoped by the stop function. """
+
+        while not stop_event.is_set():
+            reply = self.prompt_runner_for_event()
+
+            # If we have recieved 'None' then we have already timed out so skip
+            # this loop and start again
+            if reply is None:
+                continue
+
+            try:
+                valid_event(reply)
+            except Exception as e:
+                # Were not given an event, so sleep before trying again
+                sleep(self.pause_time)
+
+
+            try:
+                self.handle(reply)
+            except Exception as e:
+                # TODO some error reporting here
+                if not isinstance(e, TypeError):
+                    raise e
+
     def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
         """Function to determine given an event defintion, if this handler can
         process it or not. Must be implemented by any child process."""
         pass
 
     def handle(self, event:Dict[str,Any])->None:
-        """Function to handle a given event. Must be implemented by any child 
+        """Function to handle a given event. May be overridden by any child
-        process."""
+        process. Note that once any handling has occured, the
-        pass
+        send_job_to_runner function should be called to inform the runner of
+        any resultant jobs."""
+        rule = event[EVENT_RULE]
+
+        # Assemble job parameters dict from pattern variables
+        yaml_dict = {}
+        for var, val in rule.pattern.parameters.items():
+            yaml_dict[var] = val
+        for var, val in rule.pattern.outputs.items():
+            yaml_dict[var] = val
+        # yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH]
+
+        # If no parameter sweeps, then one job will suffice
+        if not rule.pattern.sweep:
+            self.setup_job(event, yaml_dict)
+        else:
+            # If parameter sweeps, then many jobs created
+            values_list = rule.pattern.expand_sweeps()
+            for values in values_list:
+                for value in values:
+                    yaml_dict[value[0]] = value[1]
+                self.setup_job(event, yaml_dict)
+
+    def setup_job(self, event:Dict[str,Any], params_dict:Dict[str,Any])->None:
+        """Function to set up new job dict and send it to the runner to be
+        executed."""
+
+        # Get base job metadata
+        meow_job = self.create_job_metadata_dict(event, params_dict)
+
+        # Get updated job parameters
+        # TODO replace this with generic implementation
+        from meow_base.patterns.file_event_pattern import WATCHDOG_BASE
+        params_dict = replace_keywords(
+            params_dict,
+            meow_job[JOB_ID],
+            event[EVENT_PATH],
+            event[WATCHDOG_BASE]
+        )
+
+        # Create a base job directory
+        job_dir = os.path.join(self.job_queue_dir, meow_job[JOB_ID])
+        make_dir(job_dir)
+
+        # Create job metadata file
+        meta_file = self.create_job_meta_file(job_dir, meow_job)
+
+        # Create job recipe file
+        recipe_command = self.create_job_recipe_file(job_dir, event, params_dict)
+
+        # Create job script file
+        script_command = self.create_job_script_file(job_dir, recipe_command)
+
+        threadsafe_update_status(
+            {
+                # TODO make me not tmp variables and update job dict validation
+                "tmp recipe command": recipe_command,
+                "tmp script command": script_command
+            },
+            meta_file
+        )
+
+        # Send job directory, as actual definitons will be read from within it
+        self.send_job_to_runner(job_dir)
+
+    def get_created_job_type(self)->str:
+        pass # Must implemented
+
+    def create_job_metadata_dict(self, event:Dict[str,Any],
+            params_dict:Dict[str,Any])->Dict[str,Any]:
+        return create_job_metadata_dict(
+            self.get_created_job_type(),
+            event,
+            extras={
+                JOB_PARAMETERS:params_dict
+            }
+        )
+
+    def create_job_meta_file(self, job_dir:str, meow_job:Dict[str,Any]
+            )->Dict[str,Any]:
+        meta_file = os.path.join(job_dir, META_FILE)
+
+        threadsafe_write_status(meow_job, meta_file)
+
+        return meta_file
+
+    def create_job_recipe_file(self, job_dir:str, event:Dict[str,Any], params_dict:Dict[str,Any]
+            )->str:
+        pass # Must implemented
+
+    def create_job_script_file(self, job_dir:str, recipe_command:str)->str:
+        # TODO Make this more generic, so only checking hashes if that is present
+        job_script = [
+            "#!/bin/bash",
+            "",
+            "# Get job params",
+            f"given_hash=$(grep '{WATCHDOG_HASH}: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)",
+            f"event_path=$(grep '{EVENT_PATH}: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)",
+            "",
+            "echo event_path: $event_path",
+            "echo given_hash: $given_hash",
+            "",
+            "# Check hash of input file to avoid race conditions",
+            "actual_hash=$(sha256sum $event_path | cut -c -64)",
+            "echo actual_hash: $actual_hash",
+            "if [ \"$given_hash\" != \"$actual_hash\" ]; then",
+            "   echo Job was skipped as triggering file has been modified since scheduling",
+            "   exit 134",
+            "fi",
+            "",
+            "# Call actual job script",
+            recipe_command,
+            "",
+            "exit $?"
+        ]
+        job_file = os.path.join(job_dir, JOB_FILE)
+        write_file(lines_to_string(job_script), job_file)
+        os.chmod(job_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH )
+
+        return os.path.join(".", JOB_FILE)

core/base_monitor.py

@@ -7,52 +7,64 @@ Author(s): David Marchant
 """
 
 from copy import deepcopy
-from typing import Union, Dict
+from threading import Lock
+from typing import Union, Dict, List
 
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_rule import BaseRule
+from meow_base.core.rule import Rule
-from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
+from meow_base.core.vars import VALID_CHANNELS, \
-from core.correctness.validation import check_implementation
+    VALID_MONITOR_NAME_CHARS, get_drt_imp_msg 
-from functionality.meow import create_rules
+from meow_base.functionality.validation import check_implementation, \
+    valid_string, check_type, check_types, valid_dict_multiple_types
+from meow_base.functionality.meow import create_rules, create_rule
+from meow_base.functionality.naming import generate_monitor_id
 
 
 class BaseMonitor:
+    # An identifier for a monitor within the runner. Can be manually set in 
+    # the constructor, or autogenerated if no name provided.
+    name:str
     # A collection of patterns
     _patterns: Dict[str, BasePattern]
     # A collection of recipes
     _recipes: Dict[str, BaseRecipe]
     # A collection of rules derived from _patterns and _recipes
-    _rules: Dict[str, BaseRule]
+    _rules: Dict[str, Rule]
-    # A channel for sending messages to the runner. Note that this is not 
+    # A channel for sending messages to the runner event queue. Note that this
-    # initialised within the constructor, but within the runner when passed the
+    # is not initialised within the constructor, but within the runner when the
-    # monitor is passed to it.
+    # monitor is passed to it unless the monitor is running independently of a
-    to_runner: VALID_CHANNELS
+    # runner.
+    to_runner_event: VALID_CHANNELS
+    #A lock to solve race conditions on '_patterns'
+    _patterns_lock:Lock
+    #A lock to solve race conditions on '_recipes'
+    _recipes_lock:Lock
+    #A lock to solve race conditions on '_rules'
+    _rules_lock:Lock
     def __init__(self, patterns:Dict[str,BasePattern], 
-            recipes:Dict[str,BaseRecipe])->None:
+            recipes:Dict[str,BaseRecipe], name:str="")->None:
         """BaseMonitor Constructor. This will check that any class inheriting 
         from it implements its validation functions. It will then call these on
         the input parameters."""
         check_implementation(type(self).start, BaseMonitor)
         check_implementation(type(self).stop, BaseMonitor)
-        check_implementation(type(self)._is_valid_patterns, BaseMonitor)
+        check_implementation(type(self)._get_valid_pattern_types, BaseMonitor)
         self._is_valid_patterns(patterns)
-        check_implementation(type(self)._is_valid_recipes, BaseMonitor)
+        check_implementation(type(self)._get_valid_recipe_types, BaseMonitor)
         self._is_valid_recipes(recipes)
-        check_implementation(type(self).add_pattern, BaseMonitor)
-        check_implementation(type(self).update_pattern, BaseMonitor)
-        check_implementation(type(self).remove_pattern, BaseMonitor)
-        check_implementation(type(self).get_patterns, BaseMonitor)
-        check_implementation(type(self).add_recipe, BaseMonitor)
-        check_implementation(type(self).update_recipe, BaseMonitor)
-        check_implementation(type(self).remove_recipe, BaseMonitor)
-        check_implementation(type(self).get_recipes, BaseMonitor)
-        check_implementation(type(self).get_rules, BaseMonitor)
         # Ensure that patterns and recipes cannot be trivially modified from 
         # outside the monitor, as this will cause internal consistency issues
         self._patterns = deepcopy(patterns)
         self._recipes = deepcopy(recipes)
         self._rules = create_rules(patterns, recipes)
+        if not name:
+            name = generate_monitor_id()
+        self._is_valid_name(name)
+        self.name = name
+        self._patterns_lock = Lock()
+        self._recipes_lock = Lock()
+        self._rules_lock = Lock()
         
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 
@@ -62,19 +74,152 @@ class BaseMonitor:
             raise TypeError(msg)
         return object.__new__(cls)
 
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_string(name, VALID_MONITOR_NAME_CHARS)
+
     def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
-        """Validation check for 'patterns' variable from main constructor. Must
+        """Validation check for 'patterns' variable from main constructor."""
-        be implemented by any child class."""
+        valid_dict_multiple_types(
-        pass
+            patterns, 
+            str, 
+            self._get_valid_pattern_types(), 
+            min_length=0, 
+            strict=False
+        )
+
+    def _get_valid_pattern_types(self)->List[type]:
+        """Validation check used throughout monitor to check that only 
+        compatible patterns are used. Must be implmented by any child class."""
+        raise NotImplementedError        
 
     def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
-        """Validation check for 'recipes' variable from main constructor. Must 
+        """Validation check for 'recipes' variable from main constructor."""
-        be implemented by any child class."""
+        valid_dict_multiple_types(
+            recipes, 
+            str, 
+            self._get_valid_recipe_types(), 
+            min_length=0, 
+            strict=False
+        )
+
+    def _get_valid_recipe_types(self)->List[type]:
+        """Validation check used throughout monitor to check that only 
+        compatible recipes are used. Must be implmented by any child class."""        
+        raise NotImplementedError        
+
+    def _identify_new_rules(self, new_pattern:BasePattern=None,
+            new_recipe:BaseRecipe=None)->None:
+        """Function to determine if a new rule can be created given a new 
+        pattern or recipe, in light of other existing patterns or recipes in 
+        the monitor."""
+
+        if new_pattern:
+            self._patterns_lock.acquire()
+            self._recipes_lock.acquire()
+            try:
+                # Check in case pattern has been deleted since function called
+                if new_pattern.name not in self._patterns:
+                    self._patterns_lock.release()
+                    self._recipes_lock.release()
+                    return
+                # If pattern specifies recipe that already exists, make a rule
+                if new_pattern.recipe in self._recipes:
+                    self._create_new_rule(
+                        new_pattern,
+                        self._recipes[new_pattern.recipe],
+                    )
+            except Exception as e:
+                self._patterns_lock.release()
+                self._recipes_lock.release()
+                raise e
+            self._patterns_lock.release()
+            self._recipes_lock.release()
+
+        if new_recipe:
+            self._patterns_lock.acquire()
+            self._recipes_lock.acquire()
+            try:
+                # Check in case recipe has been deleted since function called
+                if new_recipe.name not in self._recipes:
+                    self._patterns_lock.release()
+                    self._recipes_lock.release()
+                    return
+                # If recipe is specified by existing pattern, make a rule
+                for pattern in self._patterns.values():
+                    if pattern.recipe == new_recipe.name:
+                        self._create_new_rule(
+                            pattern,
+                            new_recipe,
+                        )
+            except Exception as e:
+                self._patterns_lock.release()
+                self._recipes_lock.release()
+                raise e
+            self._patterns_lock.release()
+            self._recipes_lock.release()
+
+    def _identify_lost_rules(self, lost_pattern:str=None, 
+            lost_recipe:str=None)->None:
+        """Function to remove rules that should be deleted in response to a 
+        pattern or recipe having been deleted."""
+        to_delete = []
+        self._rules_lock.acquire()
+        try:
+            # Identify any offending rules
+            for name, rule in self._rules.items():
+                if lost_pattern and rule.pattern.name == lost_pattern:
+                    to_delete.append(name)
+                if lost_recipe and rule.recipe.name == lost_recipe:
+                    to_delete.append(name)
+            # Now delete them
+            for delete in to_delete:
+                if delete in self._rules.keys():
+                    self._rules.pop(delete)
+        except Exception as e:
+            self._rules_lock.release()
+            raise e
+        self._rules_lock.release()
+
+    def _create_new_rule(self, pattern:BasePattern, recipe:BaseRecipe)->None:
+        """Function to create a new rule from a given pattern and recipe. This 
+        will only be called to create rules at runtime, as rules are 
+        automatically created at initialisation using the  same 'create_rule' 
+        function called here."""
+        rule = create_rule(pattern, recipe)
+        self._rules_lock.acquire()
+        try:
+            if rule.name in self._rules:
+                raise KeyError("Cannot create Rule with name of "
+                    f"'{rule.name}' as already in use")
+            self._rules[rule.name] = rule
+        except Exception as e:
+            self._rules_lock.release()
+            raise e
+        self._rules_lock.release()
+
+        self._apply_retroactive_rule(rule)
+
+    def _apply_retroactive_rule(self, rule:Rule)->None:
+        """Function to determine if a rule should be applied to any existing 
+        defintions, if possible. May be implemented by inherited classes."""
         pass
 
+    def _apply_retroactive_rules(self)->None:
+        """Function to determine if any rules should be applied to any existing 
+        defintions, if possible. May be implemented by inherited classes."""
+        pass
+
+    def send_event_to_runner(self, msg):
+        self.to_runner_event.send(msg)
+
     def start(self)->None:
         """Function to start the monitor as an ongoing process/thread. Must be 
-        implemented by any child process"""
+        implemented by any child process. Depending on the nature of the 
+        monitor, this may wish to directly call apply_retroactive_rules before 
+        starting."""
         pass
 
     def stop(self)->None:
@@ -83,46 +228,162 @@ class BaseMonitor:
         pass
 
     def add_pattern(self, pattern:BasePattern)->None:
-        """Function to add a pattern to the current definitions. Must be 
+        """Function to add a pattern to the current definitions. Any rules 
-        implemented by any child process."""
+        that can be possibly created from that pattern will be automatically 
-        pass
+        created."""
+        check_types(
+            pattern, 
+            self._get_valid_pattern_types(),
+             hint="add_pattern.pattern"
+        )
+
+        self._patterns_lock.acquire()
+        try:
+            if pattern.name in self._patterns:
+                raise KeyError(f"An entry for Pattern '{pattern.name}' "
+                    "already exists. Do you intend to update instead?")
+            self._patterns[pattern.name] = pattern
+        except Exception as e:
+            self._patterns_lock.release()
+            raise e            
+        self._patterns_lock.release()
+
+        self._identify_new_rules(new_pattern=pattern)
 
     def update_pattern(self, pattern:BasePattern)->None:
-        """Function to update a pattern in the current definitions. Must be 
+        """Function to update a pattern in the current definitions. Any rules 
-        implemented by any child process."""
+        created from that pattern will be automatically updated."""
-        pass
+        check_types(
+            pattern, 
+            self._get_valid_pattern_types(),
+             hint="update_pattern.pattern"
+        )
 
-    def remove_pattern(self, pattern:Union[str,BasePattern])->None:
+        self.remove_pattern(pattern.name)
-        """Function to remove a pattern from the current definitions. Must be 
+        self.add_pattern(pattern)
-        implemented by any child process."""
+
-        pass
+    def remove_pattern(self, pattern: Union[str,BasePattern])->None:
+        """Function to remove a pattern from the current definitions. Any rules 
+        that will be no longer valid will be automatically removed."""
+        check_type(
+            pattern, 
+            str, 
+            alt_types=[BasePattern], 
+            hint="remove_pattern.pattern"
+        )
+        lookup_key = pattern
+        if isinstance(lookup_key, BasePattern):
+            lookup_key = pattern.name
+        self._patterns_lock.acquire()
+        try:
+            if lookup_key not in self._patterns:
+                raise KeyError(f"Cannot remote Pattern '{lookup_key}' as it "
+                    "does not already exist")
+            self._patterns.pop(lookup_key)
+        except Exception as e:
+            self._patterns_lock.release()
+            raise e 
+        self._patterns_lock.release()
+
+        if isinstance(pattern, BasePattern):
+            self._identify_lost_rules(lost_pattern=pattern.name)
+        else:
+            self._identify_lost_rules(lost_pattern=pattern)
         
     def get_patterns(self)->Dict[str,BasePattern]:
-        """Function to get a dictionary of all current pattern definitions. 
+        """Function to get a dict of the currently defined patterns of the 
-        Must be implemented by any child process."""
+        monitor. Note that the result is deep-copied, and so can be manipulated
-        pass
+        without directly manipulating the internals of the monitor."""
+        to_return = {}
+        self._patterns_lock.acquire()
+        try:
+            to_return = deepcopy(self._patterns)
+        except Exception as e:
+            self._patterns_lock.release()
+            raise e
+        self._patterns_lock.release()
+        return to_return
 
-    def add_recipe(self, recipe:BaseRecipe)->None:
+    def add_recipe(self, recipe: BaseRecipe)->None:
-        """Function to add a recipe to the current definitions. Must be 
+        """Function to add a recipe to the current definitions. Any rules 
-        implemented by any child process."""
+        that can be possibly created from that recipe will be automatically 
-        pass
+        created."""
+        check_type(recipe, BaseRecipe, hint="add_recipe.recipe")
+        self._recipes_lock.acquire()
+        try:
+            if recipe.name in self._recipes:
+                raise KeyError(f"An entry for Recipe '{recipe.name}' already "
+                    "exists. Do you intend to update instead?")
+            self._recipes[recipe.name] = recipe
+        except Exception as e:
+            self._recipes_lock.release()
+            raise e
+        self._recipes_lock.release()
 
-    def update_recipe(self, recipe:BaseRecipe)->None:
+        self._identify_new_rules(new_recipe=recipe)
-        """Function to update a recipe in the current definitions. Must be 
+
-        implemented by any child process."""
+    def update_recipe(self, recipe: BaseRecipe)->None:
-        pass
+        """Function to update a recipe in the current definitions. Any rules 
+        created from that recipe will be automatically updated."""
+        check_type(recipe, BaseRecipe, hint="update_recipe.recipe")
+        self.remove_recipe(recipe.name)
+        self.add_recipe(recipe)
     
     def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
-        """Function to remove a recipe from the current definitions. Must be 
+        """Function to remove a recipe from the current definitions. Any rules 
-        implemented by any child process."""
+        that will be no longer valid will be automatically removed."""
-        pass
+        check_type(
+            recipe, 
+            str, 
+            alt_types=[BaseRecipe], 
+            hint="remove_recipe.recipe"
+        )
+        lookup_key = recipe
+        if isinstance(lookup_key, BaseRecipe):
+            lookup_key = recipe.name
+        self._recipes_lock.acquire()
+        try:
+            # Check that recipe has not already been deleted
+            if lookup_key not in self._recipes:
+                raise KeyError(f"Cannot remote Recipe '{lookup_key}' as it "
+                    "does not already exist")
+            self._recipes.pop(lookup_key)
+        except Exception as e:
+            self._recipes_lock.release()
+            raise e
+        self._recipes_lock.release()
+
+        if isinstance(recipe, BaseRecipe):
+            self._identify_lost_rules(lost_recipe=recipe.name)
+        else:
+            self._identify_lost_rules(lost_recipe=recipe)
 
     def get_recipes(self)->Dict[str,BaseRecipe]:
-        """Function to get a dictionary of all current recipe definitions. 
+        """Function to get a dict of the currently defined recipes of the 
-        Must be implemented by any child process."""
+        monitor. Note that the result is deep-copied, and so can be manipulated
-        pass
+        without directly manipulating the internals of the monitor."""
+        to_return = {}
+        self._recipes_lock.acquire()
+        try:
+            to_return = deepcopy(self._recipes)
+        except Exception as e:
+            self._recipes_lock.release()
+            raise e
+        self._recipes_lock.release()
+        return to_return
+    
+    def get_rules(self)->Dict[str,Rule]:
+        """Function to get a dict of the currently defined rules of the 
+        monitor. Note that the result is deep-copied, and so can be manipulated
+        without directly manipulating the internals of the monitor."""
+        to_return = {}
+        self._rules_lock.acquire()
+        try:
+            to_return = deepcopy(self._rules)
+        except Exception as e:
+            self._rules_lock.release()
+            raise e
+        self._rules_lock.release()
+        return to_return
+
 
-    def get_rules(self)->Dict[str,BaseRule]:
-        """Function to get a dictionary of all current rule definitions. 
-        Must be implemented by any child process."""
-        pass

core/base_pattern.py

@@ -10,9 +10,9 @@ import itertools
 
 from typing import Any, Union, Tuple, Dict, List
 
-from core.correctness.vars import get_drt_imp_msg, \
+from meow_base.core.vars import VALID_PATTERN_NAME_CHARS, \
-    VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
+    SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
-from core.correctness.validation import valid_string, check_type, \
+from meow_base.functionality.validation import valid_string, check_type, \
     check_implementation, valid_dict
 
 
@@ -59,7 +59,11 @@ class BasePattern:
         """Validation check for 'name' variable from main constructor. Is 
         automatically called during initialisation. This does not need to be 
         overridden by child classes."""
-        valid_string(name, VALID_PATTERN_NAME_CHARS)
+        valid_string(
+            name, 
+            VALID_PATTERN_NAME_CHARS, 
+            hint="BasePattern.name"
+        )
 
     def _is_valid_recipe(self, recipe:Any)->None:
         """Validation check for 'recipe' variable from main constructor. Must 

core/base_recipe.py

@@ -8,8 +8,10 @@ Author(s): David Marchant
 
 from typing import Any, Dict
 
-from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
+from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \
-from core.correctness.validation import valid_string, check_implementation
+    get_drt_imp_msg
+from meow_base.functionality.validation import check_implementation, \
+    valid_string
 
 
 class BaseRecipe:

core/base_rule.py

@@ -1,84 +0,0 @@
-
-"""
-This file contains the base MEOW rule defintion. This should be inherited from 
-for all rule instances.
-
-Author(s): David Marchant
-"""
-
-from sys import modules
-from typing import Any
-
-if "BasePattern" not in modules:
-    from core.base_pattern import BasePattern
-if "BaseRecipe" not in modules:
-    from core.base_recipe import BaseRecipe
-from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
-from core.correctness.validation import valid_string, check_type, \
-    check_implementation
-
-
-class BaseRule:
-    # A unique identifier for the rule
-    name:str
-    # A pattern to be used in rule triggering
-    pattern:BasePattern
-    # A recipe to be used in rule execution
-    recipe:BaseRecipe
-    # The string name of the pattern class that can be used to create this rule
-    pattern_type:str=""
-    # The string name of the recipe class that can be used to create this rule
-    recipe_type:str=""
-    def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
-        """BaseRule Constructor. This will check that any class inheriting 
-        from it implements its validation functions. It will then call these on
-        the input parameters."""
-        check_implementation(type(self)._is_valid_pattern, BaseRule)
-        check_implementation(type(self)._is_valid_recipe, BaseRule)
-        self.__check_types_set()
-        self._is_valid_name(name)
-        self.name = name
-        self._is_valid_pattern(pattern)
-        self.pattern = pattern
-        self._is_valid_recipe(recipe)
-        self.recipe = recipe
-        check_type(pattern, BasePattern, hint="BaseRule.pattern")
-        check_type(recipe, BaseRecipe, hint="BaseRule.recipe")
-        if pattern.recipe != recipe.name:
-            raise ValueError(f"Cannot create Rule {name}. Pattern "
-                f"{pattern.name} does not identify Recipe {recipe.name}. It "
-                f"uses {pattern.recipe}")
-
-    def __new__(cls, *args, **kwargs):
-        """A check that this base class is not instantiated itself, only 
-        inherited from"""
-        if cls is BaseRule:
-            msg = get_drt_imp_msg(BaseRule)
-            raise TypeError(msg)
-        return object.__new__(cls)
-
-    def _is_valid_name(self, name:str)->None:
-        """Validation check for 'name' variable from main constructor. Is 
-        automatically called during initialisation. This does not need to be 
-        overridden by child classes."""
-        valid_string(name, VALID_RULE_NAME_CHARS)
-
-    def _is_valid_pattern(self, pattern:Any)->None:
-        """Validation check for 'pattern' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def _is_valid_recipe(self, recipe:Any)->None:
-        """Validation check for 'recipe' variable from main constructor. Must 
-        be implemented by any child class."""
-        pass
-
-    def __check_types_set(self)->None:
-        """Validation check that the self.pattern_type and self.recipe_type 
-        attributes have been set in a child class."""
-        if self.pattern_type == "":
-            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
-                "does not set a pattern_type.")
-        if self.recipe_type == "":
-            raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
-                "does not set a recipe_type.")

core/correctness/validation.py

@@ -1,225 +0,0 @@
-
-"""
-This file contains various validation functions to be used throughout the 
-package.
-
-Author(s): David Marchant
-"""
-
-from inspect import signature
-from os.path import sep, exists, isfile, isdir, dirname
-from typing import Any, _SpecialForm, Union, Type, Dict, List, \
-    get_origin, get_args
-
-from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
-
-def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[], 
-        or_none:bool=False, hint:str="")->None:
-    """Checks if a given variable is of the expected type. Raises TypeError or
-    ValueError as appropriate if any issues are encountered."""
-
-    # Get a list of all allowed types
-    type_list = [expected_type]
-    if get_origin(expected_type) is Union:
-        type_list = list(get_args(expected_type))
-    type_list = type_list + alt_types
-
-    # Only accept None if explicitly allowed
-    if variable is None:
-        if or_none == False:
-            if hint:
-                msg = f"Not allowed None for {hint}. Expected {expected_type}."
-            else:
-                msg = f"Not allowed None. Expected {expected_type}."
-            raise TypeError(msg)
-        else:
-            return
-
-    # If any type is allowed, then we can stop checking
-    if expected_type == Any:
-        return
-
-    # Check that variable type is within the accepted type list
-    if not isinstance(variable, tuple(type_list)):
-        if hint:
-            msg = f"Expected type(s) for {hint} are '{type_list}', " \
-                  f"got {type(variable)}"
-        else:
-            msg = f"Expected type(s) are '{type_list}', got {type(variable)}"
-        raise TypeError(msg)
-
-def check_callable(call:Any)->None:
-    """Checks if a given variable is a callable function. Raises TypeError if 
-    not."""
-    if not callable(call):
-        raise TypeError(f"Given object '{call}' is not a callable function")
-
-def check_implementation(child_func, parent_class):
-    """Checks if the given function has been overridden from the one inherited
-    from the parent class. Raises a NotImplementedError if this is the case."""
-    # Check parent first implements func to measure against
-    if not hasattr(parent_class, child_func.__name__):
-        raise AttributeError(
-            f"Parent class {parent_class} does not implement base function "
-            f"{child_func.__name__} for children to override.")
-    parent_func = getattr(parent_class, child_func.__name__)
-
-    # Check child implements function with correct name
-    if (child_func == parent_func):
-        msg = get_not_imp_msg(parent_class, parent_func)
-        raise NotImplementedError(msg)
-
-    # Check that child implements function with correct signature
-    child_sig = signature(child_func).parameters
-    parent_sig = signature(parent_func).parameters
-    if child_sig.keys() != parent_sig.keys():
-        msg = get_not_imp_msg(parent_class, parent_func)
-        raise NotImplementedError(msg)
-
-def check_script(script:Any):
-    """Checks if a given variable is a valid script. Raises TypeError if 
-    not."""
-    # TODO investigate more robust check here
-    check_type(script, list)
-    for line in script:
-        check_type(line, str)
-
-def valid_string(variable:str, valid_chars:str, min_length:int=1)->None:
-    """Checks that all characters in a given string are present in a provided
-    list of characters. Will raise an ValueError if unexpected character is
-    encountered."""
-    check_type(variable, str)
-    check_type(valid_chars, str)
-
-    # Check string is long enough
-    if len(variable) < min_length:
-        raise ValueError (
-            f"String '{variable}' is too short. Minimum length is {min_length}"
-        )
-
-    # Check each char is acceptable
-    for char in variable:
-        if char not in valid_chars:
-            raise ValueError(
-                "Invalid character '%s'. Only valid characters are: "
-                "%s" % (char, valid_chars)
-            )
-
-def valid_dict(variable:Dict[Any, Any], key_type:Type, value_type:Type, 
-        required_keys:List[Any]=[], optional_keys:List[Any]=[], 
-        strict:bool=True, min_length:int=1)->None:
-    """Checks that a given dictionary is valid. Key and Value types are 
-    enforced, as are required and optional keys. Will raise ValueError, 
-    TypeError or KeyError depending on the problem encountered."""
-    # Validate inputs
-    check_type(variable, Dict)
-    check_type(key_type, Type, alt_types=[_SpecialForm])
-    check_type(value_type, Type, alt_types=[_SpecialForm])
-    check_type(required_keys, list)
-    check_type(optional_keys, list)
-    check_type(strict, bool)
-
-    # Check dict meets minimum length
-    if len(variable) < min_length:
-        raise ValueError(f"Dictionary '{variable}' is below minimum length of "
-            f"{min_length}")
-
-    # Check key and value types
-    for k, v in variable.items():
-        if key_type != Any and not isinstance(k, key_type):
-            raise TypeError(f"Key {k} had unexpected type '{type(k)}' "
-                f"rather than expected '{key_type}' in dict '{variable}'")
-        if value_type != Any and not isinstance(v, value_type):
-            raise TypeError(f"Value {v} had unexpected type '{type(v)}' "
-                f"rather than expected '{value_type}' in dict '{variable}'")
-
-    # Check all required keys present
-    for rk in required_keys:
-        if rk not in variable.keys():
-            raise KeyError(f"Missing required key '{rk}' from dict "
-                f"'{variable}'")
-    
-    # If strict checking, enforce that only required and optional keys are 
-    # present
-    if strict:
-        for k in variable.keys():
-            if k not in required_keys and k not in optional_keys:
-                raise ValueError(f"Unexpected key '{k}' should not be present "
-                    f"in dict '{variable}'")
-
-def valid_list(variable:List[Any], entry_type:Type,
-        alt_types:List[Type]=[], min_length:int=1, hint:str="")->None:
-    """Checks that a given list is valid. Value types are checked and a 
-    ValueError or TypeError is raised if a problem is encountered."""
-    check_type(variable, List, hint=hint)
-
-    # Check length meets minimum
-    if len(variable) < min_length:
-        raise ValueError(f"List '{variable}' is too short. Should be at least "
-            f"of length {min_length}")
-    
-    # Check type of each value
-    for n, entry in enumerate(variable):
-        if hint:
-            check_type(entry, entry_type, alt_types=alt_types,
-                hint=f"{hint}[{n}]")
-        else:
-            check_type(entry, entry_type, alt_types=alt_types)
-
-def valid_path(variable:str, allow_base:bool=False, extension:str="", 
-        min_length:int=1):
-    """Check that a given string expresses a valid path."""
-    valid_string(variable, VALID_PATH_CHARS, min_length=min_length)
-    
-    # Check we aren't given a root path
-    if not allow_base and variable.startswith(sep):
-        raise ValueError(f"Cannot accept path '{variable}'. Must be relative.")
-    
-    # Check path contains a valid extension
-    if extension and not variable.endswith(extension):
-        raise ValueError(f"Path '{variable}' does not have required "
-            f"extension '{extension}'.")
-
-def valid_existing_file_path(variable:str, allow_base:bool=False, 
-        extension:str=""):
-    """Check the given string is a path to an existing file."""
-    # Check that the string is a path
-    valid_path(variable, allow_base=allow_base, extension=extension)
-    # Check the path exists
-    if not exists(variable):
-        raise FileNotFoundError(
-            f"Requested file path '{variable}' does not exist.")
-    # Check it is a file
-    if not isfile(variable):
-        raise ValueError(
-            f"Requested file '{variable}' is not a file.")
-
-def valid_dir_path(variable:str, must_exist:bool=False, allow_base:bool=False
-        )->None:
-    """Check the given string is a valid directory path, either to an existing 
-    one or a location that could contain one."""
-    # Check that the string is a path
-    valid_path(variable, allow_base=allow_base, extension="")
-    # Check the path exists
-    does_exist = exists(variable)
-    if must_exist and not does_exist:
-        raise FileNotFoundError(
-            f"Requested dir path '{variable}' does not exist.")
-    # Check it is a directory
-    if does_exist and not isdir(variable):
-        raise ValueError(
-            f"Requested dir '{variable}' is not a directory.")
-
-def valid_non_existing_path(variable:str, allow_base:bool=False):
-    """Check the given string is a path to something that does not exist."""
-    # Check that the string is a path
-    valid_path(variable, allow_base=allow_base, extension="")
-    # Check the path does not exist
-    if exists(variable):
-        raise ValueError(f"Requested path '{variable}' already exists.")
-    # Check that any intermediate directories exist
-    if dirname(variable) and not exists(dirname(variable)):
-        raise ValueError(
-            f"Route to requested path '{variable}' does not exist.")
-
-# TODO add validation for requirement functions

core/meow.py

@@ -1,25 +1,25 @@
 
+"""
+This file contains certain meow specific defintions, most notably the 
+dictionaries passed around within the runner.
+
+Author(s): David Marchant
+"""
 from datetime import datetime
 from typing import Any, Dict, Type
 
-from core.base_rule import BaseRule
+from meow_base.core.rule import Rule
-from core.correctness.validation import check_type
+from meow_base.functionality.validation import check_type
-from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
+from meow_base.core.vars import EVENT_TYPE, EVENT_PATH, \
-    JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \
+    JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
-    JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
+    JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, EVENT_TIME
 
 # Required keys in event dict
 EVENT_KEYS = {
     EVENT_TYPE: str,
     EVENT_PATH: str,
-    # Should be a Rule but can't import here due to circular dependencies
+    EVENT_TIME: float,
-    EVENT_RULE: BaseRule
+    EVENT_RULE: Rule
-}
-
-WATCHDOG_EVENT_KEYS = {
-    WATCHDOG_BASE: str,
-    WATCHDOG_HASH: str,    
-    **EVENT_KEYS
 }
 
 # Required keys in job dict
@@ -54,6 +54,3 @@ def valid_event(event:Dict[str,Any])->None:
 def valid_job(job:Dict[str,Any])->None:
     """Check that a given dict expresses a meow job."""
     valid_meow_dict(job, "Job", JOB_KEYS)
-
-def valid_watchdog_event(event:Dict[str,Any])->None:
-    valid_meow_dict(event, "Watchdog event", WATCHDOG_EVENT_KEYS)

core/rule.py

@@ -0,0 +1,49 @@
+
+"""
+This file contains the MEOW rule defintion.
+
+Author(s): David Marchant
+"""
+
+from sys import modules
+from typing import Any
+
+if "BasePattern" not in modules:
+    from meow_base.core.base_pattern import BasePattern
+if "BaseRecipe" not in modules:
+    from meow_base.core.base_recipe import BaseRecipe
+from meow_base.core.vars import VALID_RULE_NAME_CHARS, \
+    get_drt_imp_msg
+from meow_base.functionality.validation import valid_string, check_type, \
+    check_implementation
+from meow_base.functionality.naming import generate_rule_id
+
+
+class Rule:
+    # A unique identifier for the rule
+    name:str
+    # A pattern to be used in rule triggering
+    pattern:BasePattern
+    # A recipe to be used in rule execution
+    recipe:BaseRecipe
+    def __init__(self, pattern:BasePattern, recipe:BaseRecipe, name:str=""):
+        """Rule Constructor. This will check that any class inheriting 
+        from it implements its validation functions. It will then call these on
+        the input parameters."""
+        if not name:
+            name = generate_rule_id()
+        self._is_valid_name(name)
+        self.name = name
+        check_type(pattern, BasePattern, hint="Rule.pattern")
+        self.pattern = pattern
+        check_type(recipe, BaseRecipe, hint="Rule.recipe")
+        self.recipe = recipe
+        if pattern.recipe != recipe.name:
+            raise ValueError(f"Cannot create Rule {name}. Pattern "
+                f"{pattern.name} does not identify Recipe {recipe.name}. It "
+                f"uses {pattern.recipe}")
+
+    def _is_valid_name(self, name:str)->None:
+        """Validation check for 'name' variable from main constructor. Is 
+        automatically called during initialisation."""
+        valid_string(name, VALID_RULE_NAME_CHARS)

core/runner.py

@@ -11,19 +11,20 @@ import sys
 import threading
 
 from multiprocessing import Pipe
-from random import randrange
+from typing import Any, Union, Dict, List, Type, Tuple
-from typing import Any, Union, Dict, List
 
-from core.base_conductor import BaseConductor
+from meow_base.core.base_conductor import BaseConductor
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.base_monitor import BaseMonitor
+from meow_base.core.base_monitor import BaseMonitor
-from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
+from meow_base.core.vars import DEBUG_WARNING, DEBUG_INFO, \
     VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR, \
-    EVENT_PATH
+    JOB_STATUS, STATUS_QUEUED
-from core.correctness.validation import check_type, valid_list, valid_dir_path
+from meow_base.functionality.validation import check_type, valid_list, \
-from functionality.debug import setup_debugging, print_debug
+    valid_dir_path, check_implementation
-from functionality.file_io import make_dir, read_yaml
+from meow_base.functionality.debug import setup_debugging, print_debug
-from functionality.process_io import wait
+from meow_base.functionality.file_io import make_dir, threadsafe_read_status, \
+    threadsafe_update_status
+from meow_base.functionality.process_io import wait
 
 
 class MeowRunner:
@@ -33,14 +34,18 @@ class MeowRunner:
     handlers:List[BaseHandler]
     # A collection of all conductors in the runner
     conductors:List[BaseConductor]
-    # A collection of all channels from each monitor
+    # A collection of all inputs for the event queue
-    from_monitors: List[VALID_CHANNELS]
+    event_connections: List[Tuple[VALID_CHANNELS,Union[BaseMonitor,BaseHandler]]]
-    # A collection of all channels from each handler
+    # A collection of all inputs for the job queue
-    from_handlers: List[VALID_CHANNELS]
+    job_connections: List[Tuple[VALID_CHANNELS,Union[BaseHandler,BaseConductor]]]
     # Directory where queued jobs are initially written to
     job_queue_dir:str
     # Directory where completed jobs are finally written to
     job_output_dir:str
+    # A queue of all events found by monitors, awaiting handling by handlers
+    event_queue:List[Dict[str,Any]]
+    # A queue of all jobs setup by handlers, awaiting execution by conductors
+    job_queue:List[str]
     def __init__(self, monitors:Union[BaseMonitor,List[BaseMonitor]], 
             handlers:Union[BaseHandler,List[BaseHandler]], 
             conductors:Union[BaseConductor,List[BaseConductor]],
@@ -54,6 +59,37 @@ class MeowRunner:
         self._is_valid_job_queue_dir(job_queue_dir)
         self._is_valid_job_output_dir(job_output_dir)
 
+        self.job_connections = []
+        self.event_connections = []
+
+        self._is_valid_monitors(monitors)
+        # If monitors isn't a list, make it one
+        if not type(monitors) == list:
+            monitors = [monitors]
+        self.monitors = monitors
+        for monitor in self.monitors:
+            # Create a channel from the monitor back to this runner
+            monitor_to_runner_reader, monitor_to_runner_writer = Pipe()
+            monitor.to_runner_event = monitor_to_runner_writer
+            self.event_connections.append((monitor_to_runner_reader, monitor))
+
+        self._is_valid_handlers(handlers)
+        # If handlers isn't a list, make it one
+        if not type(handlers) == list:
+            handlers = [handlers]
+        for handler in handlers:            
+            handler.job_queue_dir = job_queue_dir
+
+            # Create channels from the handler back to this runner
+            h_to_r_event_runner, h_to_r_event_handler = Pipe(duplex=True)
+            h_to_r_job_reader, h_to_r_job_writer = Pipe()
+
+            handler.to_runner_event = h_to_r_event_handler
+            handler.to_runner_job = h_to_r_job_writer
+            self.event_connections.append((h_to_r_event_runner, handler))
+            self.job_connections.append((h_to_r_job_reader, handler))
+        self.handlers = handlers
+
         self._is_valid_conductors(conductors)
         # If conductors isn't a list, make it one
         if not type(conductors) == list:
@@ -62,33 +98,13 @@ class MeowRunner:
             conductor.job_output_dir = job_output_dir
             conductor.job_queue_dir = job_queue_dir
 
+            # Create a channel from the conductor back to this runner
+            c_to_r_job_runner, c_to_r_job_conductor = Pipe(duplex=True)
+
+            conductor.to_runner_job = c_to_r_job_conductor
+            self.job_connections.append((c_to_r_job_runner, conductor))
         self.conductors = conductors
 
-        self._is_valid_handlers(handlers)
-        # If handlers isn't a list, make it one
-        if not type(handlers) == list:
-            handlers = [handlers]
-        self.from_handlers = []
-        for handler in handlers:            
-            # Create a channel from the handler back to this runner
-            handler_to_runner_reader, handler_to_runner_writer = Pipe()
-            handler.to_runner = handler_to_runner_writer
-            handler.job_queue_dir = job_queue_dir
-            self.from_handlers.append(handler_to_runner_reader)
-        self.handlers = handlers
-
-        self._is_valid_monitors(monitors)
-        # If monitors isn't a list, make it one
-        if not type(monitors) == list:
-            monitors = [monitors]
-        self.monitors = monitors
-        self.from_monitors = []
-        for monitor in self.monitors:
-            # Create a channel from the monitor back to this runner
-            monitor_to_runner_reader, monitor_to_runner_writer = Pipe()
-            monitor.to_runner = monitor_to_runner_writer
-            self.from_monitors.append(monitor_to_runner_reader)
-
         # Create channel to send stop messages to monitor/handler thread
         self._stop_mon_han_pipe = Pipe()
         self._mon_han_worker = None
@@ -100,11 +116,16 @@ class MeowRunner:
         # Setup debugging
         self._print_target, self.debug_level = setup_debugging(print, logging)
 
+        # Setup queues
+        self.event_queue = []
+        self.job_queue = []
+
     def run_monitor_handler_interaction(self)->None:
         """Function to be run in its own thread, to handle any inbound messages
         from monitors. These will be events, which should be matched to an 
         appropriate handler and handled."""
-        all_inputs = self.from_monitors + [self._stop_mon_han_pipe[0]]
+        all_inputs = [i[0] for i in self.event_connections] \
+                     + [self._stop_mon_han_pipe[0]]
         while True:
             ready = wait(all_inputs)
 
@@ -112,43 +133,45 @@ class MeowRunner:
             if self._stop_mon_han_pipe[0] in ready:
                 return
             else:
-                for from_monitor in self.from_monitors:
+                for connection, component in self.event_connections:
-                    if from_monitor in ready:
+                    if connection not in ready:
-                        # Read event from the monitor channel
+                        continue
-                        message = from_monitor.recv()
+                    message = connection.recv()
-                        event = message
 
-                        valid_handlers = []
+                    # Recieved an event
-                        for handler in self.handlers:
+                    if isinstance(component, BaseMonitor):
+                        self.event_queue.append(message)
+                        continue
+                    # Recieved a request for an event
+                    if isinstance(component, BaseHandler):
+                        valid = False
+                        for event in self.event_queue:
                             try:
-                                valid, _ = handler.valid_handle_criteria(event)
+                                valid, _ = component.valid_handle_criteria(event)
-                                if valid:
-                                    valid_handlers.append(handler)
                             except Exception as e:
                                 print_debug(
                                     self._print_target, 
                                     self.debug_level, 
-                                    "Could not determine validity of event "
+                                    "Could not determine validity of "
-                                    f"for handler. {e}", 
+                                    f"event for handler {component.name}. {e}", 
                                     DEBUG_INFO
                                 )
 
-                        # If we've only one handler, use that
+                            if valid:
-                        if len(valid_handlers) == 1:
+                                self.event_queue.remove(event)
-                            handler = valid_handlers[0]
+                                connection.send(event)
-                            self.handle_event(handler, event)
+                                break
-                        # If multiple handlers then randomly pick one
+                        
-                        else:
+                        # If nothing valid then send a message
-                            handler = valid_handlers[
+                        if not valid:
-                                randrange(len(valid_handlers))
+                            connection.send(1)
-                            ]
-                            self.handle_event(handler, event)
 
     def run_handler_conductor_interaction(self)->None:
         """Function to be run in its own thread, to handle any inbound messages
         from handlers. These will be jobs, which should be matched to an 
         appropriate conductor and executed."""
-        all_inputs = self.from_handlers + [self._stop_han_con_pipe[0]]
+        all_inputs = [i[0] for i in self.job_connections] \
+                     + [self._stop_han_con_pipe[0]]
         while True:
             ready = wait(all_inputs)
 
@@ -156,87 +179,58 @@ class MeowRunner:
             if self._stop_han_con_pipe[0] in ready:
                 return
             else:
-                for from_handler in self.from_handlers:
+                for connection, component in self.job_connections:
-                    if from_handler in ready:
+                    if connection not in ready:
-                        # Read job directory from the handler channel
-                        job_dir = from_handler.recv()
-                        try:
-                            metafile = os.path.join(job_dir, META_FILE)
-                            job = read_yaml(metafile)
-                        except Exception as e:
-                            print_debug(self._print_target, self.debug_level, 
-                                "Could not load necessary job definitions for "
-                                f"job at '{job_dir}'. {e}", DEBUG_INFO)
                         continue
 
-                        valid_conductors = []
+                    message = connection.recv()
-                        for conductor in self.conductors:
+
+                    # Recieved a job
+                    if isinstance(component, BaseHandler):
+                        self.job_queue.append(message)
+                        threadsafe_update_status(
+                            {
+                                JOB_STATUS: STATUS_QUEUED
+                            },
+                            os.path.join(message, META_FILE)
+                        )
+                        continue
+                    # Recieved a request for a job
+                    if isinstance(component, BaseConductor):
+                        valid = False
+                        print(f"Got request for job")
+                        for job_dir in self.job_queue:
                             try:
-                                valid, _ = \
+                                metafile = os.path.join(job_dir, META_FILE)
-                                    conductor.valid_execute_criteria(job)
+                                job = threadsafe_read_status(metafile)
+                            except Exception as e:
+                                print_debug(
+                                    self._print_target,
+                                    self.debug_level, 
+                                    "Could not load necessary job definitions "
+                                    f"for job at '{job_dir}'. {e}", 
+                                    DEBUG_INFO
+                                )
+
+                            try:
+                                valid, _ = component.valid_execute_criteria(job)
+                            except Exception as e:
+                                print_debug(
+                                    self._print_target,
+                                    self.debug_level, 
+                                    "Could not determine validity of "
+                                    f"job for conductor {component.name}. {e}", 
+                                    DEBUG_INFO
+                                )
+                            
                             if valid:
-                                    valid_conductors.append(conductor)
+                                self.job_queue.remove(job_dir)
-                            except Exception as e:
+                                connection.send(job_dir)
-                                print_debug(
+                                break
-                                    self._print_target, 
-                                    self.debug_level, 
-                                    "Could not determine validity of job "
-                                    f"for conductor. {e}", 
-                                    DEBUG_INFO
-                                )
 
-                        # If we've only one conductor, use that
+                        # If nothing valid then send a message
-                        if len(valid_conductors) == 1:
+                        if not valid:
-                            conductor = valid_conductors[0]
+                            connection.send(1)
-                            self.execute_job(conductor, job_dir)
-                        # If multiple handlers then randomly pick one
-                        else:
-                            conductor = valid_conductors[
-                                randrange(len(valid_conductors))
-                            ]
-                            self.execute_job(conductor, job_dir)
-   
-    def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None:
-        """Function for a given handler to handle a given event, without 
-        crashing the runner in the event of a problem."""
-        print_debug(self._print_target, self.debug_level, 
-            f"Starting handling for {event[EVENT_TYPE]} event: "
-            f"'{event[EVENT_PATH]}'", DEBUG_INFO)
-        try:
-            handler.handle(event)
-            print_debug(self._print_target, self.debug_level, 
-                f"Completed handling for {event[EVENT_TYPE]} event: "
-                f"'{event[EVENT_PATH]}'", DEBUG_INFO)
-        except Exception as e:
-            print_debug(self._print_target, self.debug_level, 
-                f"Something went wrong during handling for {event[EVENT_TYPE]}"
-                f" event '{event[EVENT_PATH]}'. {e}", DEBUG_INFO)
-
-    def execute_job(self, conductor:BaseConductor, job_dir:str)->None:
-        """Function for a given conductor to execute a given job, without 
-        crashing the runner in the event of a problem."""
-        job_id = os.path.basename(job_dir)
-        print_debug(
-            self._print_target, 
-            self.debug_level, 
-            f"Starting execution for job: '{job_id}'", 
-            DEBUG_INFO
-        )
-        try:
-            conductor.execute(job_dir)
-            print_debug(
-                self._print_target, 
-                self.debug_level, 
-                f"Completed execution for job: '{job_id}'", 
-                DEBUG_INFO
-            )
-        except Exception as e:
-            print_debug(
-                self._print_target, 
-                self.debug_level, 
-                f"Something went wrong in execution of job '{job_id}'. {e}", 
-                DEBUG_INFO
-            )
     
     def start(self)->None:
         """Function to start the runner by starting all of the constituent 
@@ -245,17 +239,14 @@ class MeowRunner:
         # Start all monitors
         for monitor in self.monitors:
             monitor.start()
-        startable = []
+
-        # Start all handlers, if they need it
+        # Start all handlers
         for handler in self.handlers:
-            if hasattr(handler, "start") and handler not in startable:
+            handler.start()
-                startable.append()
+
-        # Start all conductors, if they need it
+        # Start all conductors
         for conductor in self.conductors:
-            if hasattr(conductor, "start") and conductor not in startable:
+            conductor.start()
-                startable.append()
-        for starting in startable:
-            starting.start()
         
         # If we've not started the monitor/handler interaction thread yet, then
         # do so
@@ -295,21 +286,18 @@ class MeowRunner:
         """Function to stop the runner by stopping all of the constituent 
         monitors, handlers and conductors, along with managing interaction 
         threads."""
+
         # Stop all the monitors
         for monitor in self.monitors:
             monitor.stop()
 
-        stopable = []
         # Stop all handlers, if they need it
         for handler in self.handlers:
-            if hasattr(handler, "stop") and handler not in stopable:
+            handler.stop()
-                stopable.append()
+
         # Stop all conductors, if they need it
         for conductor in self.conductors:
-            if hasattr(conductor, "stop") and conductor not in stopable:
+            conductor.stop()
-                stopable.append()
-        for stopping in stopable:
-            stopping.stop()
 
         # If we've started the monitor/handler interaction thread, then stop it
         if self._mon_han_worker is None:
@@ -336,6 +324,60 @@ class MeowRunner:
         print_debug(self._print_target, self.debug_level,
             "Job conductor thread stopped", DEBUG_INFO)
 
+    def get_monitor_by_name(self, queried_name:str)->BaseMonitor:
+        """Gets a runner monitor with a name matching the queried name. Note 
+        in the case of multiple monitors having the same name, only the first 
+        match is returned."""
+        return self._get_entity_by_name(queried_name, self.monitors)
+
+    def get_monitor_by_type(self, queried_type:Type)->BaseMonitor:
+        """Gets a runner monitor with a type matching the queried type. Note 
+        in the case of multiple monitors having the same name, only the first 
+        match is returned."""
+        return self._get_entity_by_type(queried_type, self.monitors)
+
+    def get_handler_by_name(self, queried_name:str)->BaseHandler:
+        """Gets a runner handler with a name matching the queried name. Note 
+        in the case of multiple handlers having the same name, only the first 
+        match is returned."""
+        return self._get_entity_by_name(queried_name, self.handlers)
+
+    def get_handler_by_type(self, queried_type:Type)->BaseHandler:
+        """Gets a runner handler with a type matching the queried type. Note 
+        in the case of multiple handlers having the same name, only the first 
+        match is returned."""
+        return self._get_entity_by_type(queried_type, self.handlers)
+
+    def get_conductor_by_name(self, queried_name:str)->BaseConductor:
+        """Gets a runner conductor with a name matching the queried name. Note 
+        in the case of multiple conductors having the same name, only the first 
+        match is returned."""
+        return self._get_entity_by_name(queried_name, self.conductors)
+
+    def get_conductor_by_type(self, queried_type:Type)->BaseConductor:
+        """Gets a runner conductor with a type matching the queried type. Note 
+        in the case of multiple conductors having the same name, only the first 
+        match is returned."""
+        return self._get_entity_by_type(queried_type, self.conductors)
+
+    def _get_entity_by_name(self, queried_name:str, 
+            entities:List[Union[BaseMonitor,BaseHandler,BaseConductor]]
+            )->Union[BaseMonitor,BaseHandler,BaseConductor]:
+        """Base function inherited by more specific name query functions."""
+        for entity in entities:
+            if entity.name == queried_name:
+                return entity
+        return None
+
+    def _get_entity_by_type(self, queried_type:Type, 
+            entities:List[Union[BaseMonitor,BaseHandler,BaseConductor]]
+            )->Union[BaseMonitor,BaseHandler,BaseConductor]:
+        """Base function inherited by more specific type query functions."""
+        for entity in entities:
+            if isinstance(entity, queried_type):
+                return entity
+        return None
+
     def _is_valid_monitors(self, 
             monitors:Union[BaseMonitor,List[BaseMonitor]])->None:
         """Validation check for 'monitors' variable from main constructor."""

core/vars.py

@@ -21,6 +21,9 @@ CHAR_NUMERIC = '0123456789'
 
 VALID_NAME_CHARS = CHAR_UPPERCASE + CHAR_LOWERCASE + CHAR_NUMERIC + "_-"
 
+VALID_CONDUCTOR_NAME_CHARS = VALID_NAME_CHARS
+VALID_HANDLER_NAME_CHARS = VALID_NAME_CHARS
+VALID_MONITOR_NAME_CHARS = VALID_NAME_CHARS
 VALID_RECIPE_NAME_CHARS = VALID_NAME_CHARS
 VALID_PATTERN_NAME_CHARS = VALID_NAME_CHARS
 VALID_RULE_NAME_CHARS = VALID_NAME_CHARS
@@ -41,12 +44,8 @@ SHA256 = "sha256"
 # meow events
 EVENT_TYPE = "event_type"
 EVENT_PATH = "event_path"
-EVENT_RULE = "rule"
+EVENT_RULE = "event_rule"
-
+EVENT_TIME = "event_time"
-# watchdog events
-EVENT_TYPE_WATCHDOG = "watchdog"
-WATCHDOG_BASE = "monitor_base"
-WATCHDOG_HASH = "file_hash"
 
 # inotify events
 FILE_CREATE_EVENT = "file_created"
@@ -82,7 +81,9 @@ DEFAULT_JOB_QUEUE_DIR = "job_queue"
 DEFAULT_JOB_OUTPUT_DIR = "job_output"
 
 # meow jobs
+JOB_FILE = "job.sh"
 JOB_TYPE = "job_type"
+JOB_TYPE_BASH = "bash"
 JOB_TYPE_PYTHON = "python"
 JOB_TYPE_PAPERMILL = "papermill"
 PYTHON_FUNC = "func"
@@ -91,12 +92,17 @@ JOB_TYPES = {
     JOB_TYPE_PAPERMILL: [
         "base.ipynb",
         "job.ipynb",
-        "result.ipynb",
+        "result.ipynb"
     ],
     JOB_TYPE_PYTHON: [
         "base.py",
         "job.py",
-        "result.py",
+        "result.py"
+    ],
+    JOB_TYPE_BASH: [
+        "base.sh",
+        "job.sh",
+        "result.sh"
     ]
 }
 
@@ -116,6 +122,7 @@ JOB_REQUIREMENTS = "requirements"
 JOB_PARAMETERS = "parameters"
 
 # job statuses
+STATUS_CREATING = "creating"
 STATUS_QUEUED = "queued"
 STATUS_RUNNING = "running"
 STATUS_SKIPPED = "skipped"
@@ -136,6 +143,9 @@ DEBUG_ERROR = 1
 DEBUG_WARNING = 2
 DEBUG_INFO = 3
 
+# Locking
+LOCK_EXT = ".lock"
+
 # debug message functions
 def get_drt_imp_msg(base_class):
     return f"{base_class.__name__} may not be instantiated directly. " \
@@ -145,12 +155,3 @@ def get_not_imp_msg(parent_class, class_function):
     return f"Children of the '{parent_class.__name__}' class must implement " \
         f"the '{class_function.__name__}({signature(class_function)})' " \
         "function"
-
-def get_base_file(job_type:str):
-    return JOB_TYPES[job_type][0]
-
-def get_job_file(job_type:str):
-    return JOB_TYPES[job_type][1]
-
-def get_result_file(job_type:str):
-    return JOB_TYPES[job_type][2]

example_workflow/job_output/job_FQQUQMTGtqUw/job.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Get job params
+given_hash=$(grep 'file_hash: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)
+event_path=$(grep 'event_path: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)
+
+echo event_path: $event_path
+echo given_hash: $given_hash
+
+# Check hash of input file to avoid race conditions
+actual_hash=$(sha256sum $event_path | cut -c -64)
+echo actual_hash: $actual_hash
+if [ "$given_hash" != "$actual_hash" ]; then
+   echo Job was skipped as triggering file has been modified since scheduling
+   exit 134
+fi
+
+# Call actual job script
+python3 job_queue/job_FQQUQMTGtqUw/recipe.py >>job_queue/job_FQQUQMTGtqUw/output.log 2>&1
+
+exit $?

example_workflow/job_output/job_FQQUQMTGtqUw/job.yml

@@ -0,0 +1,38 @@
+create: 2023-06-07 11:54:48.117191
+end: 2023-06-07 11:54:53.231092
+error: Job execution returned non-zero.
+event:
+  event_path: /tmp/tmp3q4q94ee
+  event_rule: !!python/object:meow_base.core.rule.Rule
+    name: rule_xjGHQxaaray
+    pattern: !!python/object:meow_base.patterns.network_event_pattern.NetworkEventPattern
+      name: echo_pattern
+      outputs: {}
+      parameters: {}
+      recipe: echo_recipe
+      sweep: {}
+      triggering_port: 8080
+    recipe: !!python/object:meow_base.recipes.python_recipe.PythonRecipe
+      name: echo_recipe
+      parameters: {}
+      recipe:
+      - path = {PATH}
+      - print(path)
+      requirements: &id001 {}
+  event_time: 1686131685.3071685
+  event_type: network
+  file_hash: f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2
+  monitor_base: ''
+  triggering_port: 8080
+id: job_FQQUQMTGtqUw
+job_type: python
+parameters: {}
+pattern: echo_pattern
+recipe: echo_recipe
+requirements: *id001
+rule: rule_xjGHQxaaray
+start: 2023-06-07 11:54:53.168581
+status: failed
+tmp recipe command: python3 job_queue/job_FQQUQMTGtqUw/recipe.py >>job_queue/job_FQQUQMTGtqUw/output.log
+  2>&1
+tmp script command: ./job.sh

example_workflow/job_output/job_FQQUQMTGtqUw/recipe.py

@@ -0,0 +1,2 @@
+path = {PATH}
+print(path)

example_workflow/network_workflow.py

@@ -0,0 +1,53 @@
+from time import sleep
+
+from meow_base.core.runner import MeowRunner
+from meow_base.patterns.network_event_pattern import NetworkMonitor, NetworkEventPattern
+from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler
+from meow_base.conductors.local_python_conductor import LocalPythonConductor
+
+PORTS = [8080,8181]
+
+def main():
+    runners = []
+    for i, port in enumerate(PORTS):
+        other_port = PORTS[(i+1)%2]
+
+        # Gets the script ready
+        script = [
+            "import socket",
+            "sender = socket.socket(socket.AF_INET, socket.SOCK_STREAM)",
+            f"sender.connect(('127.0.0.1', {other_port}))",
+            "sender.sendall(b'test')",
+            "sender.close()"
+        ]
+
+        # Initialize the network monitor
+        patterns = {
+            "echo_pattern":NetworkEventPattern(
+                "echo_pattern",
+                port,
+                "echo_recipe"
+            )
+        }
+        recipes = {"echo_recipe":PythonRecipe("echo_recipe", script)}
+        monitors = [NetworkMonitor(patterns, recipes)]
+
+
+        # Initialize the handler and conductor
+        handlers = [PythonHandler()]
+        conductors = [LocalPythonConductor()]
+
+        # Start the runner
+        runner = MeowRunner(monitors, handlers, conductors)
+
+        runner.start()
+        runners.append(runner)
+
+    sleep(120)
+
+    for runner in runners:
+        runner.stop()
+
+
+if __name__ == "__main__":
+    main()

functionality/debug.py

@@ -6,8 +6,8 @@ Author(s): David Marchant
 
 from typing import Any, Tuple
 
-from core.correctness.validation import check_type
+from meow_base.functionality.validation import check_type
-from core.correctness.vars import DEBUG_INFO, DEBUG_WARNING
+from meow_base.core.vars import DEBUG_INFO, DEBUG_WARNING
 
 
 def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:

functionality/file_io.py

@@ -4,6 +4,7 @@ This file contains functions for reading and writing different types of files.
 Author(s): David Marchant
 """
 
+import fcntl
 import json
 import yaml
 
@@ -11,7 +12,10 @@ from os import makedirs, remove, rmdir, walk
 from os.path import exists, isfile, join
 from typing import Any, Dict, List
 
-from core.correctness.validation import valid_path
+from meow_base.core.vars import JOB_END_TIME, JOB_ERROR, JOB_STATUS, \
+    STATUS_FAILED, STATUS_DONE, JOB_CREATE_TIME, JOB_START_TIME, \
+    STATUS_SKIPPED, LOCK_EXT
+from meow_base.functionality.validation import valid_path
 
 
 def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):
@@ -94,6 +98,68 @@ def write_yaml(source:Any, filename:str):
     with open(filename, 'w') as param_file:
         yaml.dump(source, param_file, default_flow_style=False)
 
+def threadsafe_read_status(filepath:str):
+    lock_path = filepath + LOCK_EXT
+    lock_handle = open(lock_path, 'a')
+    fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
+
+    try:
+        status = read_yaml(filepath)
+    except Exception as e:
+        lock_handle.close()
+        raise e
+
+    lock_handle.close()
+
+    return status
+
+def threadsafe_write_status(source:dict[str,Any], filepath:str):
+    lock_path = filepath + LOCK_EXT
+    lock_handle = open(lock_path, 'a')
+    fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
+
+    try:
+        write_yaml(source, filepath)
+    except Exception as e:
+        lock_handle.close()
+        raise e
+
+    lock_handle.close()
+
+def threadsafe_update_status(updates:dict[str,Any], filepath:str):
+    lock_path = filepath + LOCK_EXT
+    lock_handle = open(lock_path, 'a')
+    fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
+
+    try:
+        status = read_yaml(filepath)
+
+        for k, v in status.items():
+            if k in updates:
+                # Do not overwrite final job status
+                if k == JOB_STATUS \
+                        and v in [STATUS_DONE, STATUS_FAILED, STATUS_SKIPPED]:
+                    continue
+                # Do not overwrite an existing time
+                elif k in [JOB_START_TIME, JOB_CREATE_TIME, JOB_END_TIME]:
+                    continue
+                # Do not overwrite an existing error messages
+                elif k == JOB_ERROR:
+                    updates[k] = f"{v} {updates[k]}"
+
+                status[k] = updates[k]
+            
+        for k, v in updates.items():
+            if k not in status:
+                status[k] = v
+
+        write_yaml(status, filepath)
+    except Exception as e:
+        lock_handle.close()
+        raise e
+
+    lock_handle.close()
+
 def read_notebook(filepath:str):
     valid_path(filepath, extension="ipynb")
     with open(filepath, 'r') as read_file:

functionality/hashing.py

@@ -5,11 +5,15 @@ Author(s): David Marchant
 """
 
 from hashlib import sha256
+from os import listdir
+from os.path import isfile
 
-from core.correctness.vars import HASH_BUFFER_SIZE, SHA256
-from core.correctness.validation import check_type, valid_existing_file_path
 
-def _get_file_sha256(file_path):
+from meow_base.core.vars import HASH_BUFFER_SIZE, SHA256
+from meow_base.functionality.validation import check_type, \
+    valid_existing_file_path, valid_existing_dir_path
+
+def _get_file_sha256(file_path:str)->str:
     sha256_hash = sha256()
     
     with open(file_path, 'rb') as file_to_hash:
@@ -21,7 +25,16 @@ def _get_file_sha256(file_path):
     
     return sha256_hash.hexdigest()
 
-def get_file_hash(file_path:str, hash:str, hint:str=""):
+# TODO update this to be a bit more robust
+def _get_dir_sha256(dir_path:str)->str:
+    sha256_hash = sha256()
+
+    buffer = str(listdir(dir_path)).encode()
+    sha256_hash.update(buffer)
+
+    return sha256_hash.hexdigest()
+
+def get_file_hash(file_path:str, hash:str, hint:str="")->str:
     check_type(hash, str, hint=hint)
 
     valid_existing_file_path(file_path)
@@ -34,3 +47,24 @@ def get_file_hash(file_path:str, hash:str, hint:str=""):
             f"'{list(valid_hashes.keys())}")
 
     return valid_hashes[hash](file_path)
+
+# TODO inspect this a bit more fully 
+def get_dir_hash(file_path:str, hash:str, hint:str="")->str:
+    check_type(hash, str, hint=hint)
+
+    valid_existing_dir_path(file_path)
+
+    valid_hashes = {
+        SHA256: _get_dir_sha256
+    }
+    if hash not in valid_hashes:
+        raise KeyError(f"Cannot use hash '{hash}'. Valid are "
+            f"'{list(valid_hashes.keys())}")
+
+    return valid_hashes[hash](file_path)
+
+def get_hash(path:str, hash:str, hint:str="")->str:
+    if isfile(path):
+        return get_file_hash(path, hash, hint=hint)
+    else:
+        return get_dir_hash(path, hash, hint=hint)

functionality/meow.py

@@ -8,16 +8,16 @@ from datetime import datetime
 from os.path import basename, dirname, relpath, splitext
 from typing import Any, Dict, Union, List
 
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_rule import BaseRule
+from meow_base.core.rule import Rule
-from core.correctness.validation import check_type, valid_dict, valid_list
+from meow_base.functionality.validation import check_type, valid_dict, \
-from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
+    valid_list
-    EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \
+from meow_base.core.vars import EVENT_PATH, EVENT_RULE, EVENT_TIME, \
-    JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \
+    EVENT_TYPE, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \
-    STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, SWEEP_START, \
+    JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
-    SWEEP_STOP
+    JOB_TYPE, STATUS_CREATING, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
-from functionality.naming import generate_job_id, generate_rule_id
+from meow_base.functionality.naming import generate_job_id
 
 # mig trigger keyword replacements
 KEYWORD_PATH = "{PATH}"
@@ -31,6 +31,8 @@ KEYWORD_EXTENSION = "{EXTENSION}"
 KEYWORD_JOB = "{JOB}"
 
 
+# TODO make this generic for all event types, currently very tied to file 
+# events
 def replace_keywords(old_dict:Dict[str,str], job_id:str, src_path:str, 
             monitor_base:str)->Dict[str,str]:
     """Function to replace all MEOW magic words in a dictionary with dynamic 
@@ -101,34 +103,19 @@ def create_parameter_sweep(variable_name:str, start:Union[int,float,complex],
         }
     }
 
-def create_event(event_type:str, path:str, rule:Any, extras:Dict[Any,Any]={}
+def create_event(event_type:str, path:str, rule:Any, time:float,
-        )->Dict[Any,Any]:
+        extras:Dict[Any,Any]={})->Dict[Any,Any]:
     """Function to create a MEOW dictionary."""
     return {
         **extras, 
         EVENT_PATH: path, 
         EVENT_TYPE: event_type, 
-        EVENT_RULE: rule
+        EVENT_RULE: rule,
+        EVENT_TIME: time
     }
 
-def create_watchdog_event(path:str, rule:Any, base:str, hash:str, 
+def create_job_metadata_dict(job_type:str, event:Dict[str,Any], 
         extras:Dict[Any,Any]={})->Dict[Any,Any]:
-    """Function to create a MEOW event dictionary."""
-    return create_event(
-        EVENT_TYPE_WATCHDOG, 
-        path, 
-        rule,
-        extras={
-            **extras,
-            **{
-                WATCHDOG_HASH: hash,
-                WATCHDOG_BASE: base
-            }
-        }
-    )
-
-def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
-        )->Dict[Any,Any]:
     """Function to create a MEOW job dictionary."""
     job_dict = {
         #TODO compress event?
@@ -138,7 +125,7 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
         JOB_PATTERN: event[EVENT_RULE].pattern.name,
         JOB_RECIPE: event[EVENT_RULE].recipe.name,
         JOB_RULE: event[EVENT_RULE].name,
-        JOB_STATUS: STATUS_QUEUED,
+        JOB_STATUS: STATUS_CREATING,
         JOB_CREATE_TIME: datetime.now(),
         JOB_REQUIREMENTS: event[EVENT_RULE].recipe.requirements
     }
@@ -146,8 +133,7 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
     return {**extras, **job_dict}
 
 def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], 
-        recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
+        recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]])->Dict[str,Rule]:
-        new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
     """Function to create any valid rules from a given collection of patterns 
     and recipes. All inbuilt rule types are considered, with additional 
     definitions provided through the 'new_rules' variable. Note that any 
@@ -156,7 +142,6 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
     # Validation of inputs
     check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns")
     check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes")
-    valid_list(new_rules, BaseRule, min_length=0)
 
     # Convert a pattern list to a dictionary
     if isinstance(patterns, list):
@@ -197,34 +182,14 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
                 pass
     return generated_rules
 
-def create_rule(pattern:BasePattern, recipe:BaseRecipe, 
+def create_rule(pattern:BasePattern, recipe:BaseRecipe)->Rule:
-        new_rules:List[BaseRule]=[])->BaseRule:
     """Function to create a valid rule from a given pattern and recipe. All 
     inbuilt rule types are considered, with additional definitions provided 
     through the 'new_rules' variable."""
     check_type(pattern, BasePattern, hint="create_rule.pattern")
     check_type(recipe, BaseRecipe, hint="create_rule.recipe")
-    valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules")
     
-    # TODO fix me
+    return Rule(
-    # Imported here to avoid circular imports at top of file
-    import rules
-    all_rules = {
-        (r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()
-    }
-
-    # Add in new rules
-    for rule in new_rules:
-        all_rules[(rule.pattern_type, rule.recipe_type)] = rule
-
-    # Find appropriate rule type from pattern and recipe types
-    key = (type(pattern).__name__, type(recipe).__name__)
-    if (key) in all_rules:
-        return all_rules[key](
-            generate_rule_id(), 
         pattern, 
         recipe
     )
-    # Raise error if not valid rule type can be found
-    raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe "
-        f"'{recipe}' could be found.")

functionality/naming.py

@@ -7,7 +7,7 @@ Author(s): David Marchant
 from typing import List
 from random import SystemRandom
 
-from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
+from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
 
 
 #TODO Make this guaranteed unique
@@ -27,3 +27,12 @@ def generate_rule_id():
 
 def generate_job_id():
     return _generate_id(prefix="job_")
+
+def generate_conductor_id():
+    return _generate_id(prefix="conductor_")
+
+def generate_handler_id():
+    return _generate_id(prefix="handler_")
+
+def generate_monitor_id():
+    return _generate_id(prefix="monitor_")

functionality/parameterisation.py

@@ -10,7 +10,7 @@ from os import getenv
 from papermill.translators import papermill_translators
 from typing import Any, Dict, List
 
-from core.correctness.validation import check_script, check_type
+from meow_base.functionality.validation import check_script, check_type
 
 # Adapted from: https://github.com/rasmunk/notebook_parameterizer
 def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any], 
@@ -119,3 +119,36 @@ def parameterize_python_script(script:List[str], parameters:Dict[str,Any],
     check_script(output_script)
 
     return output_script
+
+def parameterize_bash_script(script:List[str], parameters:Dict[str,Any], 
+        expand_env_values:bool=False)->Dict[str,Any]:
+    check_script(script)
+    check_type(parameters, Dict
+        ,hint="parameterize_bash_script.parameters")
+
+    output_script = deepcopy(script)
+
+    for i, line in enumerate(output_script):
+        if "=" in line:
+            d_line = list(map(lambda x: x.replace(" ", ""), 
+                line.split("=")))
+            # Matching parameter name
+            if len(d_line) == 2 and d_line[0] in parameters:
+                value = parameters[d_line[0]]
+                # Whether to expand value from os env
+                if (
+                    expand_env_values
+                    and isinstance(value, str)
+                    and value.startswith("ENV_")
+                ):
+                    env_var = value.replace("ENV_", "")
+                    value = getenv(
+                        env_var, 
+                        "MISSING ENVIRONMENT VARIABLE: {}".format(env_var)
+                    )
+                output_script[i] = f"{d_line[0]}={repr(value)}"
+                
+    # Validate that the parameterized notebook is still valid
+    check_script(output_script)
+
+    return output_script

functionality/process_io.py

@@ -12,7 +12,7 @@ from multiprocessing.connection import Connection, wait as multi_wait
 if osName == 'nt':
     from multiprocessing.connection import PipeConnection
 from multiprocessing.queues import Queue
-from core.correctness.vars import VALID_CHANNELS
+from meow_base.core.vars import VALID_CHANNELS
 
 
 def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:

functionality/requirements.py

@@ -11,7 +11,7 @@ from os.path import basename
 from sys import version_info, prefix, base_prefix
 from typing import Any, Dict, List, Tuple, Union
 
-from core.correctness.validation import check_type
+from meow_base.functionality.validation import check_type
 
 REQUIREMENT_PYTHON = "python"
 REQ_PYTHON_MODULES = "modules"

functionality/validation.py

@@ -0,0 +1,371 @@
+
+"""
+This file contains various validation functions to be used throughout the 
+package.
+
+Author(s): David Marchant
+"""
+
+from inspect import signature
+from os.path import sep, exists, isfile, isdir, dirname
+from typing import Any, _SpecialForm, Union, Type, Dict, List, \
+    get_origin, get_args
+
+from meow_base.core.vars import VALID_PATH_CHARS, get_not_imp_msg
+
+def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[], 
+        or_none:bool=False, hint:str="")->None:
+    """Checks if a given variable is of the expected type. Raises TypeError or
+    ValueError as appropriate if any issues are encountered."""
+
+    # Get a list of all allowed types
+    type_list = [expected_type]
+    if get_origin(expected_type) is Union:
+        type_list = list(get_args(expected_type))
+    type_list = type_list + alt_types
+
+    # Only accept None if explicitly allowed
+    if variable is None:
+        if or_none == False:
+            if hint:
+                msg = f"Not allowed None for {hint}. Expected {expected_type}."
+            else:
+                msg = f"Not allowed None. Expected {expected_type}."
+            raise TypeError(msg)
+        else:
+            return
+
+    # If any type is allowed, then we can stop checking
+    if expected_type == Any:
+        return
+
+    # Check that variable type is within the accepted type list
+    if not isinstance(variable, tuple(type_list)):
+        if hint:
+            msg = f"Expected type(s) for {hint} are '{type_list}', " \
+                  f"got {type(variable)}"
+        else:
+            msg = f"Expected type(s) are '{type_list}', got {type(variable)}"
+        raise TypeError(msg)
+
+def check_types(variable:Any, expected_types:List[Type], or_none:bool=False, 
+        hint:str="")->None:
+    """Checks if a given variable is one of the expected types. Raises 
+    TypeError or ValueError as appropriate if any issues are encountered."""
+    check_type(variable, expected_types[0], alt_types=expected_types[1:],
+                or_none=or_none, hint=hint)
+
+def check_callable(call:Any, hint:str="")->None:
+    """Checks if a given variable is a callable function. Raises TypeError if 
+    not."""
+    if not callable(call):
+        if hint:
+            raise TypeError(
+                f"Given object '{call}' by '{hint}' is not a callable function"
+            )
+        else:
+            raise TypeError(
+                f"Given object '{call}' is not a callable function"
+            )
+
+def check_implementation(child_func, parent_class):
+    """Checks if the given function has been overridden from the one inherited
+    from the parent class. Raises a NotImplementedError if this is the case."""
+    # Check parent first implements func to measure against
+    if not hasattr(parent_class, child_func.__name__):
+        raise AttributeError(
+            f"Parent class {parent_class} does not implement base function "
+            f"{child_func.__name__} for children to override.")
+    parent_func = getattr(parent_class, child_func.__name__)
+
+    # Check child implements function with correct name
+    if (child_func == parent_func):
+        msg = get_not_imp_msg(parent_class, parent_func)
+        raise NotImplementedError(msg)
+
+    # Check that child implements function with correct signature
+    child_sig = signature(child_func).parameters
+    parent_sig = signature(parent_func).parameters
+    if child_sig.keys() != parent_sig.keys():
+        msg = get_not_imp_msg(parent_class, parent_func)
+        raise NotImplementedError(msg)
+
+def check_script(script:Any):
+    """Checks if a given variable is a valid script. Raises TypeError if 
+    not."""
+    # TODO investigate more robust check here
+    check_type(script, list)
+    for line in script:
+        check_type(line, str)
+
+def valid_string(variable:str, valid_chars:str, min_length:int=1, hint:str=""
+        )->None:
+    """Checks that all characters in a given string are present in a provided
+    list of characters. Will raise an ValueError if unexpected character is
+    encountered."""
+    check_type(variable, str, hint=hint)
+    check_type(valid_chars, str, hint=hint)
+
+    # Check string is long enough
+    if len(variable) < min_length:
+        if hint:
+            msg = f"String '{variable}' for '{hint}' is too short. Minimum " \
+                  f"length is {min_length}"
+        else:
+            msg = f"String '{variable}' is too short. Minimum length is " \
+                  f"{min_length}"
+        raise ValueError (msg)
+
+    # Check each char is acceptable
+    for char in variable:
+        if char not in valid_chars:
+            if hint :
+                msg = f"Invalid character '{char}' in '{hint}'. Only valid " \
+                      f"characters are: {valid_chars}"
+            else:
+                msg = f"Invalid character '{char}'. Only valid characters " \
+                      f"are: {valid_chars}"
+            raise ValueError(msg)
+
+def valid_dict(variable:Dict[Any, Any], key_type:Type, value_type:Type, 
+        required_keys:List[Any]=[], optional_keys:List[Any]=[], 
+        strict:bool=True, min_length:int=1, hint:str="")->None:
+    """Checks that a given dictionary is valid. Key and Value types are 
+    enforced, as are required and optional keys. Will raise ValueError, 
+    TypeError or KeyError depending on the problem encountered."""
+    # Validate inputs
+    check_type(variable, Dict, hint=hint)
+    check_type(key_type, Type, alt_types=[_SpecialForm], hint=hint)
+    check_type(value_type, Type, alt_types=[_SpecialForm], hint=hint)
+    check_type(required_keys, list, hint=hint)
+    check_type(optional_keys, list, hint=hint)
+    check_type(strict, bool, hint=hint)
+
+    if hint:
+        hint = f"in '{hint}' "
+
+    # Check dict meets minimum length
+    if len(variable) < min_length:
+        raise ValueError(
+            f"Dictionary '{variable}' {hint}is below minimum length of "
+            f"{min_length}"
+        )
+
+    # Check key and value types
+    for k, v in variable.items():
+        if key_type != Any and not isinstance(k, key_type):
+            raise TypeError(f"Key {k} {hint}had unexpected type '{type(k)}' "
+                f"rather than expected '{key_type}' in dict '{variable}'")
+        if value_type != Any and not isinstance(v, value_type):
+            raise TypeError(f"Value {v} {hint}had unexpected type '{type(v)}' "
+                f"rather than expected '{value_type}' in dict '{variable}'")
+
+    # Check all required keys present
+    for rk in required_keys:
+        if rk not in variable.keys():
+            raise KeyError(f"Missing required key '{rk}' from dict "
+                f"'{variable}' {hint}.")
+    
+    # If strict checking, enforce that only required and optional keys are 
+    # present
+    if strict:
+        for k in variable.keys():
+            if k not in required_keys and k not in optional_keys:
+                raise ValueError(f"Unexpected key '{k}' {hint}should not be "
+                                 f"present in dict '{variable}'")
+
+def valid_dict_multiple_types(variable:Dict[Any, Any], key_type:Type, 
+        value_types:List[Type], required_keys:List[Any]=[], 
+        optional_keys:List[Any]=[], strict:bool=True, min_length:int=1, 
+        hint:str="")->None:
+    """Checks that a given dictionary is valid. Key and Value types are 
+    enforced, as are required and optional keys. Will raise ValueError, 
+    TypeError or KeyError depending on the problem encountered."""
+    # Validate inputs
+    check_type(variable, Dict, hint=hint)
+    check_type(key_type, Type, alt_types=[_SpecialForm], hint=hint)
+    valid_list(value_types, Type, alt_types=[_SpecialForm], hint=hint)
+    check_type(required_keys, list, hint=hint)
+    check_type(optional_keys, list, hint=hint)
+    check_type(strict, bool, hint=hint)
+
+    if hint:
+        hint = f"in '{hint}' "
+
+    # Check dict meets minimum length
+    if len(variable) < min_length:
+        raise ValueError(
+            f"Dictionary '{variable}' {hint}is below minimum length of "
+            f"{min_length}"
+        )
+
+    # Check key and value types
+    for k, v in variable.items():
+        if key_type != Any and not isinstance(k, key_type):
+            raise TypeError(f"Key {k} {hint}had unexpected type '{type(k)}' "
+                f"rather than expected '{key_type}' in dict '{variable}'")
+        if Any not in value_types and type(v) not in value_types:
+            raise TypeError(f"Value {v} {hint}had unexpected type '{type(v)}' "
+                f"rather than expected '{value_types}' in dict '{variable}'")
+
+    # Check all required keys present
+    for rk in required_keys:
+        if rk not in variable.keys():
+            raise KeyError(f"Missing required key '{rk}' from dict "
+                f"'{variable}' {hint}.")
+    
+    # If strict checking, enforce that only required and optional keys are 
+    # present
+    if strict:
+        for k in variable.keys():
+            if k not in required_keys and k not in optional_keys:
+                raise ValueError(f"Unexpected key '{k}' {hint}should not be "
+                                 f"present in dict '{variable}'")
+
+def valid_list(variable:List[Any], entry_type:Type,
+        alt_types:List[Type]=[], min_length:int=1, hint:str="")->None:
+    """Checks that a given list is valid. Value types are checked and a 
+    ValueError or TypeError is raised if a problem is encountered."""
+    check_type(variable, List, hint=hint)
+
+    # Check length meets minimum
+    if len(variable) < min_length:
+        if hint:
+            msg = f"List '{variable}' is too short in {hint}. Should be at " \
+                  f"least of length {min_length}"
+        else:
+            msg = f"List '{variable}' is too short. Should be at least " \
+                  f"of length {min_length}"
+        raise ValueError(msg)
+    
+    # Check type of each value
+    for n, entry in enumerate(variable):
+        if hint:
+            check_type(entry, entry_type, alt_types=alt_types,
+                hint=f"{hint}[{n}]")
+        else:
+            check_type(entry, entry_type, alt_types=alt_types)
+
+def valid_path(variable:str, allow_base:bool=False, extension:str="", 
+        min_length:int=1, hint:str=""):
+    """Check that a given string expresses a valid path."""
+    valid_string(variable, VALID_PATH_CHARS, min_length=min_length, hint=hint)
+    
+    # Check we aren't given a root path
+    if not allow_base and variable.startswith(sep):
+        if hint:
+            msg = f"Cannot accept path '{variable}' in '{hint}'. Must be " \
+                  "relative."
+        else:
+            msg = f"Cannot accept path '{variable}'. Must be relative."
+        raise ValueError(msg)
+    
+    # Check path contains a valid extension
+    if extension and not variable.endswith(extension):
+        if hint:
+            msg = f"Path '{variable}' in '{hint}' does not have required " \
+                  f"extension '{extension}'."
+        else:
+            msg = f"Path '{variable}' does not have required extension " \
+                  f"'{extension}'."
+        raise ValueError(msg)
+
+def valid_existing_file_path(variable:str, allow_base:bool=False, 
+        extension:str="", hint:str=""):
+    """Check the given string is a path to an existing file."""
+    # Check that the string is a path
+    valid_path(variable, allow_base=allow_base, extension=extension, hint=hint)
+    # Check the path exists
+    if not exists(variable):
+        if hint:
+            msg = f"Requested file path '{variable}' in '{hint}' does not " \
+                  "exist."
+        else:
+            msg = f"Requested file path '{variable}' does not exist."
+        raise FileNotFoundError(msg)
+    # Check it is a file
+    if not isfile(variable):
+        if hint: 
+            msg = f"Requested file '{variable}' in '{hint}' is not a file."
+        else:
+            msg = f"Requested file '{variable}' is not a file."
+        raise ValueError(msg)
+
+def valid_existing_dir_path(variable:str, allow_base:bool=False, 
+        extension:str="", hint:str=""):
+    """Check the given string is a path to an existing dir."""
+    # Check that the string is a path
+    valid_path(variable, allow_base=allow_base, extension=extension, hint=hint)
+    # Check the path exists
+    if not exists(variable):
+        if hint:
+            msg = f"Requested dir path '{variable}' in '{hint}' does not " \
+                  "exist."
+        else:
+            msg = f"Requested dir path '{variable}' does not exist."
+        raise FileNotFoundError(msg)
+    # Check it is a dir
+    if not isdir(variable):
+        if hint: 
+            msg = f"Requested dir '{variable}' in '{hint}' is not a dir."
+        else:
+            msg = f"Requested dir '{variable}' is not a dir."
+        raise ValueError(msg)
+
+def valid_dir_path(variable:str, must_exist:bool=False, allow_base:bool=False,
+        hint:str="")->None:
+    """Check the given string is a valid directory path, either to an existing 
+    one or a location that could contain one."""
+    # Check that the string is a path
+    valid_path(variable, allow_base=allow_base, extension="", hint=hint)
+    # Check the path exists
+    does_exist = exists(variable)
+    if must_exist and not does_exist:
+        if hint:
+            msg = f"Requested dir path '{variable}' in '{hint}' does not " \
+                  "exist."
+        else:
+            msg = f"Requested dir path '{variable}' does not exist."
+        raise FileNotFoundError(msg)
+    # Check it is a directory
+    if does_exist and not isdir(variable):
+        if hint:
+            msg = f"Requested dir '{variable}' in '{hint}' is not a directory."
+        else:
+            msg = f"Requested dir '{variable}' is not a directory."
+        raise ValueError()
+
+def valid_non_existing_path(variable:str, allow_base:bool=False, hint:str=""
+        )->None:
+    """Check the given string is a path to something that does not exist."""
+    # Check that the string is a path
+    valid_path(variable, allow_base=allow_base, extension="", hint=hint)
+    # Check the path does not exist
+    if exists(variable):
+        if hint:
+            msg = f"Requested path '{variable}' in '{hint}' already exists."
+        else:
+            msg = f"Requested path '{variable}' already exists."
+        raise ValueError(msg)
+    # Check that any intermediate directories exist
+    if dirname(variable) and not exists(dirname(variable)):
+        if hint:
+            msg = f"Route to requested path '{variable}' in '{hint}' does " \
+                  "not exist."
+        else:
+            msg = f"Route to requested path '{variable}' does not exist."
+        raise ValueError(msg)
+
+def valid_natural(num:int, hint:str="")->None:
+    """Check a given value is a natural number. Will raise a ValueError if not."""
+    check_type(num, int, hint=hint)
+
+    if num < 0:
+        if hint :
+            msg = f"Value {num} in {hint} is not a natural number."
+        else:
+            msg = f"Value {num} is not a natural number."
+
+        raise ValueError(msg)
+
+# TODO add validation for requirement functions

patterns/__init__.py

@@ -1,2 +1,2 @@
 
-from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor
+from .file_event_pattern import FileEventPattern, WatchdogMonitor

patterns/file_event_pattern.py

@@ -10,7 +10,6 @@ import threading
 import sys
 import os
 
-from copy import deepcopy
 from fnmatch import translate
 from re import match
 from time import time, sleep
@@ -18,19 +17,21 @@ from typing import Any, Union, Dict, List
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_monitor import BaseMonitor
+from meow_base.core.base_monitor import BaseMonitor
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_rule import BaseRule
+from meow_base.core.meow import EVENT_KEYS, valid_meow_dict
-from core.correctness.validation import check_type, valid_string, \
+from meow_base.core.rule import Rule
-    valid_dict, valid_list, valid_path, valid_dir_path
+from meow_base.functionality.validation import check_type, valid_string, \
-from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
+    valid_dict, valid_list, valid_dir_path
+from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \
     VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
-    FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
+    FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, DIR_EVENTS, \
-    FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT
+    FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT, \
-from functionality.debug import setup_debugging, print_debug
+    DIR_RETROACTIVE_EVENT
-from functionality.hashing import get_file_hash
+from meow_base.functionality.debug import setup_debugging, print_debug
-from functionality.meow import create_rule, create_watchdog_event
+from meow_base.functionality.hashing import get_hash
+from meow_base.functionality.meow import create_event
 
 # Events that are monitored by default
 _DEFAULT_MASK = [
@@ -41,6 +42,35 @@ _DEFAULT_MASK = [
     FILE_CLOSED_EVENT
 ]
 
+# watchdog events
+EVENT_TYPE_WATCHDOG = "watchdog"
+WATCHDOG_BASE = "monitor_base"
+WATCHDOG_HASH = "file_hash"
+
+WATCHDOG_EVENT_KEYS = {
+    WATCHDOG_BASE: str,
+    WATCHDOG_HASH: str,
+    **EVENT_KEYS
+}
+
+def create_watchdog_event(path:str, rule:Any, base:str, time:float,
+            hash:str, extras:Dict[Any,Any]={})->Dict[Any,Any]:
+    """Function to create a MEOW event dictionary."""
+    return create_event(
+        EVENT_TYPE_WATCHDOG,
+        path,
+        rule,
+        time,
+        extras={
+            **extras,
+            **{
+                WATCHDOG_HASH: hash,
+                WATCHDOG_BASE: base
+            }
+        }
+    )
+
+
 class FileEventPattern(BasePattern):
     # The path at which events will trigger this pattern
     triggering_path:str
@@ -65,44 +95,84 @@ class FileEventPattern(BasePattern):
     def _is_valid_triggering_path(self, triggering_path:str)->None:
         """Validation check for 'triggering_path' variable from main
         constructor."""
-        valid_string(triggering_path, VALID_PATH_CHARS+'*', min_length=1)
+        valid_string(
+            triggering_path,
+            VALID_PATH_CHARS+'*',
+            min_length=1,
+            hint="FileEventPattern.triggering_path"
+        )
         if len(triggering_path) < 1:
             raise ValueError (
-                f"triggiering path '{triggering_path}' is too short. " 
+                f"triggering path '{triggering_path}' is too short. "
                 "Minimum length is 1"
         )
 
     def _is_valid_triggering_file(self, triggering_file:str)->None:
         """Validation check for 'triggering_file' variable from main
         constructor."""
-        valid_string(triggering_file, VALID_VARIABLE_NAME_CHARS)
+        valid_string(
+            triggering_file,
+            VALID_VARIABLE_NAME_CHARS,
+            hint="FileEventPattern.triggering_file"
+        )
 
     def _is_valid_recipe(self, recipe:str)->None:
         """Validation check for 'recipe' variable from main constructor.
         Called within parent BasePattern constructor."""
-        valid_string(recipe, VALID_RECIPE_NAME_CHARS)
+        valid_string(
+            recipe,
+            VALID_RECIPE_NAME_CHARS,
+            hint="FileEventPattern.recipe"
+        )
 
     def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
         """Validation check for 'parameters' variable from main constructor.
         Called within parent BasePattern constructor."""
-        valid_dict(parameters, str, Any, strict=False, min_length=0)
+        valid_dict(
+            parameters,
+            str,
+            Any,
+            strict=False,
+            min_length=0,
+            hint="FileEventPattern.parameters"
+        )
         for k in parameters.keys():
-            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+            valid_string(
+                k,
+                VALID_VARIABLE_NAME_CHARS,
+                hint=f"FileEventPattern.parameters[{k}]"
+            )
 
     def _is_valid_output(self, outputs:Dict[str,str])->None:
         """Validation check for 'output' variable from main constructor.
         Called within parent BasePattern constructor."""
-        valid_dict(outputs, str, str, strict=False, min_length=0)
+        valid_dict(
+            outputs,
+            str,
+            str,
+            strict=False,
+            min_length=0,
+            hint="FileEventPattern.outputs"
+        )
         for k in outputs.keys():
-            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+            valid_string(
+                k,
+                VALID_VARIABLE_NAME_CHARS,
+                hint=f"FileEventPattern.outputs[{k}]"
+            )
 
     def _is_valid_event_mask(self, event_mask)->None:
         """Validation check for 'event_mask' variable from main constructor."""
-        valid_list(event_mask, str, min_length=1)
+        valid_list(
+            event_mask,
+            str,
+            min_length=1,
+            hint="FileEventPattern.event_mask"
+        )
         for mask in event_mask:
-            if mask not in FILE_EVENTS:
+            if mask not in FILE_EVENTS + DIR_EVENTS:
                 raise ValueError(f"Invalid event mask '{mask}'. Valid are: "
-                    f"{FILE_EVENTS}")
+                    f"{FILE_EVENTS + DIR_EVENTS}")
 
     def _is_valid_sweep(self, sweep: Dict[str,Union[int,float,complex]]) -> None:
         """Validation check for 'sweep' variable from main constructor."""
@@ -120,28 +190,18 @@ class WatchdogMonitor(BaseMonitor):
     debug_level:int
     # Where print messages are sent
     _print_target:Any
-    #A lock to solve race conditions on '_patterns'
-    _patterns_lock:threading.Lock
-    #A lock to solve race conditions on '_recipes'
-    _recipes_lock:threading.Lock
-    #A lock to solve race conditions on '_rules'
-    _rules_lock:threading.Lock
-
     def __init__(self, base_dir:str, patterns:Dict[str,FileEventPattern],
             recipes:Dict[str,BaseRecipe], autostart=False, settletime:int=1,
-            print:Any=sys.stdout, logging:int=0)->None:
+            name:str="", print:Any=sys.stdout, logging:int=0)->None:
         """WatchdogEventHandler Constructor. This uses the watchdog module to
         monitor a directory and all its sub-directories. Watchdog will provide
         the monitor with an caught events, with the monitor comparing them
         against its rules, and informing the runner of match."""
-        super().__init__(patterns, recipes)
+        super().__init__(patterns, recipes, name=name)
         self._is_valid_base_dir(base_dir)
         self.base_dir = base_dir
         check_type(settletime, int, hint="WatchdogMonitor.settletime")
         self._print_target, self.debug_level = setup_debugging(print, logging)
-        self._patterns_lock = threading.Lock()
-        self._recipes_lock = threading.Lock()
-        self._rules_lock = threading.Lock()
         self.event_handler = WatchdogEventHandler(self, settletime=settletime)
         self.monitor = Observer()
         self.monitor.schedule(
@@ -194,6 +254,10 @@ class WatchdogMonitor(BaseMonitor):
                 # Use regex to match event paths against rule paths
                 target_path = rule.pattern.triggering_path
                 recursive_regexp = translate(target_path)
+                if os.name == 'nt':
+                    direct_regexp = recursive_regexp.replace(
+                        '.*', '[^'+ os.path.sep + os.path.sep +']*')
+                else:
                     direct_regexp = recursive_regexp.replace(
                         '.*', '[^'+ os.path.sep +']*')
                 recursive_hit = match(recursive_regexp, handle_path)
@@ -205,13 +269,14 @@ class WatchdogMonitor(BaseMonitor):
                         event.src_path,
                         rule,
                         self.base_dir,
-                        get_file_hash(event.src_path, SHA256) 
+                        event.time_stamp,
+                        get_hash(event.src_path, SHA256)
                     )
                     print_debug(self._print_target, self.debug_level,
                         f"Event at {src_path} hit rule {rule.name}",
                         DEBUG_INFO)
                     # Send the event to the runner
-                    self.to_runner.send(meow_event)
+                    self.send_event_to_runner(meow_event)
 
         except Exception as e:
             self._rules_lock.release()
@@ -219,248 +284,6 @@ class WatchdogMonitor(BaseMonitor):
 
         self._rules_lock.release()
 
-    def add_pattern(self, pattern:FileEventPattern)->None:
-        """Function to add a pattern to the current definitions. Any rules 
-        that can be possibly created from that pattern will be automatically 
-        created."""
-        check_type(pattern, FileEventPattern, hint="add_pattern.pattern")
-        self._patterns_lock.acquire()
-        try:
-            if pattern.name in self._patterns:
-                raise KeyError(f"An entry for Pattern '{pattern.name}' "
-                    "already exists. Do you intend to update instead?")
-            self._patterns[pattern.name] = pattern
-        except Exception as e:
-            self._patterns_lock.release()
-            raise e            
-        self._patterns_lock.release()
-
-        self._identify_new_rules(new_pattern=pattern)
-
-    def update_pattern(self, pattern:FileEventPattern)->None:
-        """Function to update a pattern in the current definitions. Any rules 
-        created from that pattern will be automatically updated."""
-        check_type(pattern, FileEventPattern, hint="update_pattern.pattern")
-        self.remove_pattern(pattern.name)
-        self.add_pattern(pattern)
-
-    def remove_pattern(self, pattern: Union[str,FileEventPattern])->None:
-        """Function to remove a pattern from the current definitions. Any rules 
-        that will be no longer valid will be automatically removed."""
-        check_type(
-            pattern, 
-            str, 
-            alt_types=[FileEventPattern], 
-            hint="remove_pattern.pattern"
-        )
-        lookup_key = pattern
-        if isinstance(lookup_key, FileEventPattern):
-            lookup_key = pattern.name
-        self._patterns_lock.acquire()
-        try:
-            if lookup_key not in self._patterns:
-                raise KeyError(f"Cannot remote Pattern '{lookup_key}' as it "
-                    "does not already exist")
-            self._patterns.pop(lookup_key)
-        except Exception as e:
-            self._patterns_lock.release()
-            raise e 
-        self._patterns_lock.release()
-
-        if isinstance(pattern, FileEventPattern):
-            self._identify_lost_rules(lost_pattern=pattern.name)
-        else:
-            self._identify_lost_rules(lost_pattern=pattern)
-        
-    def get_patterns(self)->Dict[str,FileEventPattern]:
-        """Function to get a dict of the currently defined patterns of the 
-        monitor. Note that the result is deep-copied, and so can be manipulated
-        without directly manipulating the internals of the monitor."""
-        to_return = {}
-        self._patterns_lock.acquire()
-        try:
-            to_return = deepcopy(self._patterns)
-        except Exception as e:
-            self._patterns_lock.release()
-            raise e
-        self._patterns_lock.release()
-        return to_return
-
-    def add_recipe(self, recipe: BaseRecipe)->None:
-        """Function to add a recipe to the current definitions. Any rules 
-        that can be possibly created from that recipe will be automatically 
-        created."""
-        check_type(recipe, BaseRecipe, hint="add_recipe.recipe")
-        self._recipes_lock.acquire()
-        try:
-            if recipe.name in self._recipes:
-                raise KeyError(f"An entry for Recipe '{recipe.name}' already "
-                    "exists. Do you intend to update instead?")
-            self._recipes[recipe.name] = recipe
-        except Exception as e:
-            self._recipes_lock.release()
-            raise e
-        self._recipes_lock.release()
-
-        self._identify_new_rules(new_recipe=recipe)
-
-    def update_recipe(self, recipe: BaseRecipe)->None:
-        """Function to update a recipe in the current definitions. Any rules 
-        created from that recipe will be automatically updated."""
-        check_type(recipe, BaseRecipe, hint="update_recipe.recipe")
-        self.remove_recipe(recipe.name)
-        self.add_recipe(recipe)
-    
-    def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
-        """Function to remove a recipe from the current definitions. Any rules 
-        that will be no longer valid will be automatically removed."""
-        check_type(
-            recipe, 
-            str, 
-            alt_types=[BaseRecipe], 
-            hint="remove_recipe.recipe"
-        )
-        lookup_key = recipe
-        if isinstance(lookup_key, BaseRecipe):
-            lookup_key = recipe.name
-        self._recipes_lock.acquire()
-        try:
-            # Check that recipe has not already been deleted
-            if lookup_key not in self._recipes:
-                raise KeyError(f"Cannot remote Recipe '{lookup_key}' as it "
-                    "does not already exist")
-            self._recipes.pop(lookup_key)
-        except Exception as e:
-            self._recipes_lock.release()
-            raise e
-        self._recipes_lock.release()
-
-        if isinstance(recipe, BaseRecipe):
-            self._identify_lost_rules(lost_recipe=recipe.name)
-        else:
-            self._identify_lost_rules(lost_recipe=recipe)
-
-    def get_recipes(self)->Dict[str,BaseRecipe]:
-        """Function to get a dict of the currently defined recipes of the 
-        monitor. Note that the result is deep-copied, and so can be manipulated
-        without directly manipulating the internals of the monitor."""
-        to_return = {}
-        self._recipes_lock.acquire()
-        try:
-            to_return = deepcopy(self._recipes)
-        except Exception as e:
-            self._recipes_lock.release()
-            raise e
-        self._recipes_lock.release()
-        return to_return
-    
-    def get_rules(self)->Dict[str,BaseRule]:
-        """Function to get a dict of the currently defined rules of the 
-        monitor. Note that the result is deep-copied, and so can be manipulated
-        without directly manipulating the internals of the monitor."""
-        to_return = {}
-        self._rules_lock.acquire()
-        try:
-            to_return = deepcopy(self._rules)
-        except Exception as e:
-            self._rules_lock.release()
-            raise e
-        self._rules_lock.release()
-        return to_return
-
-    def _identify_new_rules(self, new_pattern:FileEventPattern=None, 
-            new_recipe:BaseRecipe=None)->None:
-        """Function to determine if a new rule can be created given a new 
-        pattern or recipe, in light of other existing patterns or recipes in 
-        the monitor."""
-
-        if new_pattern:
-            self._patterns_lock.acquire()
-            self._recipes_lock.acquire()
-            try:
-                # Check in case pattern has been deleted since function called
-                if new_pattern.name not in self._patterns:
-                    self._patterns_lock.release()
-                    self._recipes_lock.release()
-                    return
-                # If pattern specifies recipe that already exists, make a rule
-                if new_pattern.recipe in self._recipes:
-                    self._create_new_rule(
-                        new_pattern,
-                        self._recipes[new_pattern.recipe],
-                    )
-            except Exception as e:
-                self._patterns_lock.release()
-                self._recipes_lock.release()
-                raise e
-            self._patterns_lock.release()
-            self._recipes_lock.release()
-
-        if new_recipe:
-            self._patterns_lock.acquire()
-            self._recipes_lock.acquire()
-            try:
-                # Check in case recipe has been deleted since function called
-                if new_recipe.name not in self._recipes:
-                    self._patterns_lock.release()
-                    self._recipes_lock.release()
-                    return
-                # If recipe is specified by existing pattern, make a rule
-                for pattern in self._patterns.values():
-                    if pattern.recipe == new_recipe.name:
-                        self._create_new_rule(
-                            pattern,
-                            new_recipe,
-                        )
-            except Exception as e:
-                self._patterns_lock.release()
-                self._recipes_lock.release()
-                raise e
-            self._patterns_lock.release()
-            self._recipes_lock.release()
-
-    def _identify_lost_rules(self, lost_pattern:str=None, 
-            lost_recipe:str=None)->None:
-        """Function to remove rules that should be deleted in response to a 
-        pattern or recipe having been deleted."""
-        to_delete = []
-        self._rules_lock.acquire()
-        try:
-            # Identify any offending rules
-            for name, rule in self._rules.items():
-                if lost_pattern and rule.pattern.name == lost_pattern:
-                    to_delete.append(name)
-                if lost_recipe and rule.recipe.name == lost_recipe:
-                    to_delete.append(name)
-            # Now delete them
-            for delete in to_delete:
-                if delete in self._rules.keys():
-                    self._rules.pop(delete)
-        except Exception as e:
-            self._rules_lock.release()
-            raise e
-        self._rules_lock.release()
-
-    def _create_new_rule(self, pattern:FileEventPattern, 
-            recipe:BaseRecipe)->None:
-        """Function to create a new rule from a given pattern and recipe. This 
-        will only be called to create rules at runtime, as rules are 
-        automatically created at initialisation using the  same 'create_rule' 
-        function called here."""
-        rule = create_rule(pattern, recipe)
-        self._rules_lock.acquire()
-        try:
-            if rule.name in self._rules:
-                raise KeyError("Cannot create Rule with name of "
-                    f"'{rule.name}' as already in use")
-            self._rules[rule.name] = rule
-        except Exception as e:
-            self._rules_lock.release()
-            raise e
-        self._rules_lock.release()
-
-        self._apply_retroactive_rule(rule)
-        
     def _is_valid_base_dir(self, base_dir:str)->None:
         """Validation check for 'base_dir' variable from main constructor. Is
         automatically called during initialisation."""
@@ -476,13 +299,13 @@ class WatchdogMonitor(BaseMonitor):
         automatically called during initialisation."""
         valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False)
 
-    def _apply_retroactive_rules(self)->None:
+    def _get_valid_pattern_types(self)->List[type]:
-        """Function to determine if any rules should be applied to the existing 
+        return [FileEventPattern]
-        file structure, were the file structure created/modified now."""
-        for rule in self._rules.values():
-            self._apply_retroactive_rule(rule)
 
-    def _apply_retroactive_rule(self, rule:BaseRule)->None:
+    def _get_valid_recipe_types(self)->List[type]:
+        return [BaseRecipe]
+
+    def _apply_retroactive_rule(self, rule:Rule)->None:
         """Function to determine if a rule should be applied to the existing
         file structure, were the file structure created/modified now."""
         self._rules_lock.acquire()
@@ -492,7 +315,8 @@ class WatchdogMonitor(BaseMonitor):
                 self._rules_lock.release()
                 return
 
-            if FILE_RETROACTIVE_EVENT in rule.pattern.event_mask:
+            if FILE_RETROACTIVE_EVENT in rule.pattern.event_mask \
+                    or DIR_RETROACTIVE_EVENT in rule.pattern.event_mask:
                 # Determine what paths are potentially triggerable and gather
                 # files at those paths
                 testing_path = os.path.join(
@@ -507,19 +331,26 @@ class WatchdogMonitor(BaseMonitor):
                         globble,
                         rule,
                         self.base_dir,
-                        get_file_hash(globble, SHA256)
+                        time(),
+                        get_hash(globble, SHA256)
                     )
                     print_debug(self._print_target, self.debug_level,
                         f"Retroactive event for file at at {globble} hit rule "
                         f"{rule.name}", DEBUG_INFO)
                     # Send it to the runner
-                    self.to_runner.send(meow_event)
+                    self.send_event_to_runner(meow_event)
 
         except Exception as e:
             self._rules_lock.release()
             raise e
         self._rules_lock.release()
 
+    def _apply_retroactive_rules(self)->None:
+        """Function to determine if any rules should be applied to the existing
+        file structure, were the file structure created/modified now."""
+        for rule in self._rules.values():
+            self._apply_retroactive_rule(rule)
+
 
 class WatchdogEventHandler(PatternMatchingEventHandler):
     # The monitor class running this handler
@@ -559,6 +390,14 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
             else:
                 self._recent_jobs[event.src_path] = \
                     [event.time_stamp, {event.event_type}]
+
+            # If we have a closed event then short-cut the wait and send event
+            # immediately
+            if event.event_type == FILE_CLOSED_EVENT:
+                self.monitor.match(event)
+                self._recent_jobs_lock.release()
+                return
+
         except Exception as ex:
             self._recent_jobs_lock.release()
             raise Exception(ex)
@@ -580,29 +419,6 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
 
         self.monitor.match(event)
 
-#                recent_timestamp = self._recent_jobs[event.src_path]
-#                difference = event.time_stamp - recent_timestamp
-#
-#                # Discard the event if we already have a recent event at this 
-#                # same path. Update the most recent time, so we can hopefully
-#                # wait till events have stopped happening
-#                if difference <= self._settletime:
-#                    self._recent_jobs[event.src_path] = \
-#                        max(recent_timestamp, event.time_stamp)
-#                    self._recent_jobs_lock.release()
-#                    return
-#                else:
-#                    self._recent_jobs[event.src_path] = event.time_stamp
-#            else:
-#                self._recent_jobs[event.src_path] = event.time_stamp
-#        except Exception as ex:
-#            self._recent_jobs_lock.release()
-#            raise Exception(ex)
-#        self._recent_jobs_lock.release()
-#
-#        # If we did not have a recent event, then send it on to the monitor
-#        self.monitor.match(event)
-
     def handle_event(self, event):
         """Handler function, called by all specific event functions. Will
         attach a timestamp to the event immediately, and attempt to start a

patterns/network_event_pattern.py

@@ -0,0 +1,271 @@
+import sys
+import socket
+import threading
+import tempfile
+import hashlib
+from os import unlink
+from http.server import HTTPServer
+
+from time import time
+
+from typing import Any, Dict, List
+from http_parser.parser import HttpParser
+
+from meow_base.functionality.validation import valid_string, valid_dict
+from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, VALID_VARIABLE_NAME_CHARS, DEBUG_INFO
+from meow_base.core.base_recipe import BaseRecipe
+from meow_base.core.base_monitor import BaseMonitor
+from meow_base.core.base_pattern import BasePattern
+from meow_base.functionality.meow import create_event
+from meow_base.functionality.debug import setup_debugging, print_debug
+from meow_base.core.meow import EVENT_KEYS
+from meow_base.patterns.file_event_pattern import WATCHDOG_BASE, WATCHDOG_HASH
+
+# network events
+EVENT_TYPE_NETWORK = "network"
+TRIGGERING_PORT = "triggering_port"
+
+NETWORK_EVENT_KEYS = {
+    TRIGGERING_PORT: int,
+    WATCHDOG_HASH: str,
+    WATCHDOG_BASE: str,
+    **EVENT_KEYS
+}
+
+def create_network_event(temp_path:str, rule:Any, time:float,
+                         port: int, file_hash: str,
+                         extras:Dict[Any,Any]={})->Dict[Any,Any]:
+    """Function to create a MEOW event dictionary."""
+    return create_event(
+        EVENT_TYPE_NETWORK,
+        temp_path,
+        rule,
+        time,
+        extras={
+            TRIGGERING_PORT: port,
+            WATCHDOG_HASH: file_hash,
+            WATCHDOG_BASE: "",
+            **extras
+        }
+    )
+
+class NetworkEventPattern(BasePattern):
+    # The port to monitor
+    triggering_port:int
+
+    def __init__(self, name: str, triggering_port:int, recipe: str, parameters: Dict[str, Any] = {}, outputs: Dict[str, Any] = {}, sweep: Dict[str, Any] = {}):
+        super().__init__(name, recipe, parameters, outputs, sweep)
+        self._is_valid_port(triggering_port)
+        self.triggering_port = triggering_port
+
+    def _is_valid_port(self, port:int)->None:
+        if not isinstance(port, int):
+            raise ValueError (
+                f"Port '{port}' is not of type int."
+            )
+        elif not (1023 < port < 49152):
+            raise ValueError (
+                f"Port '{port}' is not valid."
+            )
+
+    def _is_valid_recipe(self, recipe:str)->None:
+        """Validation check for 'recipe' variable from main constructor.
+        Called within parent BasePattern constructor."""
+        valid_string(recipe, VALID_RECIPE_NAME_CHARS)
+
+    def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
+        """Validation check for 'parameters' variable from main constructor.
+        Called within parent BasePattern constructor."""
+        valid_dict(parameters, str, Any, strict=False, min_length=0)
+        for k in parameters.keys():
+            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+
+    def _is_valid_output(self, outputs:Dict[str,str])->None:
+        """Validation check for 'output' variable from main constructor.
+        Called within parent BasePattern constructor."""
+        valid_dict(outputs, str, str, strict=False, min_length=0)
+        for k in outputs.keys():
+            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+
+class NetworkMonitor(BaseMonitor):
+    def __init__(self, patterns: Dict[str, NetworkEventPattern],
+            recipes: Dict[str, BaseRecipe], autostart=False,
+            name:str="", print:Any=sys.stdout, logging:int=0) -> None:
+        super().__init__(patterns, recipes, name=name)
+        self._print_target, self.debug_level = setup_debugging(print, logging)
+        self.ports = set()
+        self.listeners = []
+        if not hasattr(self, "listener_type"):
+            self.listener_type = Listener
+        if autostart:
+            self.start()
+
+    def start(self)->None:
+        """Function to start the monitor as an ongoing process/thread. Must be
+        implemented by any child process. Depending on the nature of the
+        monitor, this may wish to directly call apply_retroactive_rules before
+        starting."""
+        self.temp_files = []
+
+        self.ports = set(
+            rule.pattern.triggering_port for rule in self._rules.values()
+        )
+        self.listeners = [
+            self.listener_type("127.0.0.1",i,2048,self) for i in self.ports
+        ]
+
+        for listener in self.listeners:
+            listener.start()
+
+    def match(self, event)->None:
+        """Function to determine if a given event matches the current rules."""
+
+        self._rules_lock.acquire()
+        try:
+            self.temp_files.append(event["tmp file"])
+            for rule in self._rules.values():
+                # Match event port against rule ports
+                hit = event["triggering port"] == rule.pattern.triggering_port
+
+                # If matched, the create a watchdog event
+                if hit:
+                    meow_event = create_network_event(
+                        event["tmp file"],
+                        rule,
+                        event["time stamp"],
+                        event["triggering port"],
+                        event["file hash"]
+                    )
+                    print_debug(self._print_target, self.debug_level,
+                        f"Event at {event['triggering port']} hit rule {rule.name}",
+                        DEBUG_INFO)
+                    # Send the event to the runner
+                    self.send_event_to_runner(meow_event)
+
+        except Exception as e:
+            self._rules_lock.release()
+            raise e
+
+        self._rules_lock.release()
+
+    def _delete_temp_files(self):
+        for file in self.temp_files:
+            unlink(file)
+        self.temp_files = []
+
+    def stop(self)->None:
+        """Function to stop the monitor as an ongoing process/thread. Must be
+        implemented by any child process"""
+        for listener in self.listeners:
+            listener.stop()
+
+        self._delete_temp_files()
+
+    def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
+        """Validation check for 'recipes' variable from main constructor. Is
+        automatically called during initialisation."""
+        valid_dict(recipes, str, BaseRecipe, min_length=0, strict=False)
+
+    def _get_valid_pattern_types(self)->List[type]:
+        return [NetworkEventPattern]
+
+    def _get_valid_recipe_types(self)->List[type]:
+        return [BaseRecipe]
+
+class Listener():
+    def __init__(self, host: int, port: int, buff_size: int,
+                 monitor:NetworkMonitor) -> None:
+        self.host = host
+        self.port = port
+        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.socket.settimeout(0.5)
+        self._stopped = False
+        self.buff_size = buff_size
+        self.monitor = monitor
+
+    def start(self):
+        self._handle_thread = threading.Thread(
+            target=self.main_loop
+        )
+        self._handle_thread.start()
+
+    def main_loop(self):
+        self.socket.bind((self.host, self.port))
+        self.socket.listen(1)
+        while not self._stopped:
+            try:
+                conn, _ = self.socket.accept()
+            except socket.timeout:
+                pass
+            except:
+                raise
+            else:
+                threading.Thread(
+                    target=self.handle_event,
+                    args=(conn,time(),)
+                ).start()
+
+        self.socket.close()
+
+    def receive_data(self,conn):
+        with conn:
+            with tempfile.NamedTemporaryFile("wb", delete=False) as tmp:
+                while True:
+                    data = conn.recv(self.buff_size)
+                    if not data:
+                        break
+                    tmp.write(data)
+
+                tmp_name = tmp.name
+
+        return tmp_name
+
+    def handle_event(self, conn, time_stamp):
+        tmp_name = self.receive_data(conn)
+
+        with open(tmp_name, "rb") as file_pointer:
+            file_hash = hashlib.sha256(file_pointer.read()).hexdigest()
+
+        event = {
+            "triggering port": self.port,
+            "tmp file": tmp_name,
+            "time stamp": time_stamp,
+            "file hash": file_hash
+        }
+        self.monitor.match(event)
+
+    def stop(self):
+        self._stopped = True
+
+class HTTPMonitor(NetworkMonitor):
+    def __init__(self, patterns: Dict[str, NetworkEventPattern],
+            recipes: Dict[str, BaseRecipe], autostart=False,
+            name: str = "", print: Any = sys.stdout, logging: int = 0) -> None:
+        self.listener_type = HTTPListener()
+        super().__init__(patterns, recipes, autostart, name, print, logging)
+
+class HTTPListener(Listener):
+    def receive_data(self,conn):
+        parser = HttpParser()
+
+        with conn:
+            with tempfile.NamedTemporaryFile("wb", delete=False) as tmp:
+                while True:
+                    data = conn.recv(self.buff_size)
+                    if not data:
+                        break
+
+                    received = len(data)
+                    parsed = parser.execute(data, received)
+                    assert parsed == received
+
+                    if parser.is_partial_body():
+                        tmp.write(parser.recv_body())
+
+                    if parser.is_message_complete():
+                        break
+                tmp_name = tmp.name
+
+        return tmp_name
+

performance_test/performance_test.py

@@ -0,0 +1,106 @@
+import socket
+from multiprocessing import Pipe
+from threading import Thread
+from time import time, sleep
+from numpy import std, floor, log10
+
+from meow_base.patterns.network_event_pattern import NetworkMonitor, \
+    NetworkEventPattern
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
+from meow_base.tests.shared import BAREBONES_NOTEBOOK
+
+def send(port):
+    sender = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    while True:
+        try:
+            sender.connect(("127.0.0.1", port))
+            sender.sendall(b'test')
+            sender.close()
+            break
+        except:
+            continue
+
+def get_all(from_monitor_reader, event_count):
+    for _ in range(event_count):
+        if from_monitor_reader.poll(3):
+            event = from_monitor_reader.recv()
+        else:
+            raise Exception("Did not receive all events")
+
+def test_network(monitor_count: int, patterns_per_monitor: int,
+                 events_per_pattern: int, start_port: int):
+
+    monitors = []
+    port = start_port
+
+
+    recipe = JupyterNotebookRecipe(
+        "recipe_one", BAREBONES_NOTEBOOK)
+    recipes = {
+        recipe.name: recipe
+    }
+
+    from_monitor_reader, from_monitor_writer = Pipe()
+
+    for _ in range(monitor_count):
+        patterns = {}
+        for p in range(patterns_per_monitor):
+            pattern_name = f"pattern_{p}"
+            patterns[pattern_name] = NetworkEventPattern(
+                pattern_name,
+                port,
+                recipe.name
+            )
+            port += 1
+
+        monitor = NetworkMonitor(patterns, recipes)
+        monitor.to_runner_event = from_monitor_writer
+        monitors.append(monitor)
+        monitor.start()
+
+
+    event_count = monitor_count*patterns_per_monitor*events_per_pattern
+
+    receiver = Thread(target=get_all, args=(from_monitor_reader, event_count,))
+    receiver.start()
+
+    start_time = time()
+
+
+    for p in range(start_port, port):
+        for _ in range(events_per_pattern):
+            send(p)
+            # Thread(target=send, args=(p,)).start()
+
+    receiver.join()
+
+    duration = time() - start_time
+
+    for monitor in monitors:
+        monitor.stop()
+
+    return duration
+
+def sigfigs(num):
+    if num < 10:
+        return round(num, -int(floor(log10(abs(num))-1)))
+    else:
+        return int(num)
+
+def main(monitors, patterns, events):
+    n = 100
+
+    durations = []
+    for i in range(n):
+        print(" ", i, end=" \r")
+        durations.append(test_network(monitors,patterns,events,1024))
+        sleep(0.5)
+
+    print(f"({monitors}, {patterns}, {events}) min: {min(durations)}, max: {max(durations)}, avg: {sum(durations)/n}, std: {std(durations)}")
+
+    # print(f"{sigfigs(min(durations)*1000)}ms & {sigfigs((min(durations)*1000)/events)}ms & {sigfigs(max(durations)*1000)}ms & {sigfigs((max(durations)*1000)/events)}ms & {sigfigs((sum(durations)/n)*1000)}ms & {sigfigs(((sum(durations)/n)*1000)/events)}ms & {sigfigs(std(durations)*1000)}ms")
+    print(f"{patterns} & {sigfigs(min(durations)*1000)}ms & {sigfigs(max(durations)*1000)}ms & {sigfigs((sum(durations)/n)*1000)}ms & {sigfigs(std(durations)*1000)}ms")
+
+if __name__ == "__main__":
+    for m, p, e in [(1,100,100), (1,1_000,10), (1,2_500,4), (1,5_000,2), (1,10_000,1)]:
+        main(m, p, e)

recipes/__init__.py

@@ -1,4 +1,5 @@
 
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
+from .jupyter_notebook_recipe import JupyterNotebookRecipe, PapermillHandler, \
-    PapermillHandler
+    get_recipe_from_notebook
-from recipes.python_recipe import PythonRecipe, PythonHandler
+from .python_recipe import PythonRecipe, PythonHandler
+from .bash_recipe import BashRecipe, BashHandler

recipes/bash_recipe.py

@@ -0,0 +1,116 @@
+
+import os
+import stat
+import sys
+
+from typing import Any, Dict, List, Tuple
+
+from meow_base.core.base_handler import BaseHandler
+from meow_base.core.base_recipe import BaseRecipe
+from meow_base.core.meow import valid_event
+from meow_base.functionality.validation import check_type, valid_dict, \
+    valid_string, valid_dir_path
+from meow_base.core.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
+    VALID_VARIABLE_NAME_CHARS, EVENT_RULE, EVENT_TYPE, \
+    JOB_TYPE_BASH
+from meow_base.functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.file_io import valid_path, make_dir, write_file, \
+    lines_to_string
+from meow_base.functionality.parameterisation import parameterize_bash_script
+from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG
+
+class BashRecipe(BaseRecipe):
+    # A path to the bash script used to create this recipe
+    def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, 
+            requirements:Dict[str,Any]={}, source:str=""):
+        """BashRecipe Constructor. This is used to execute bash scripts, 
+        enabling anything not natively supported by MEOW."""
+        super().__init__(name, recipe, parameters, requirements)
+        self._is_valid_source(source)
+        self.source = source
+
+    def _is_valid_source(self, source:str)->None:
+        """Validation check for 'source' variable from main constructor."""
+        if source:
+            valid_path(source, extension=".sh", min_length=0)
+
+    def _is_valid_recipe(self, recipe:List[str])->None:
+        """Validation check for 'recipe' variable from main constructor. 
+        Called within parent BaseRecipe constructor."""
+        check_type(recipe, List, hint="BashRecipe.recipe")
+        for line in recipe:
+            check_type(line, str, hint="BashRecipe.recipe[line]")
+
+    def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
+        """Validation check for 'parameters' variable from main constructor. 
+        Called within parent BaseRecipe constructor."""
+        valid_dict(parameters, str, Any, strict=False, min_length=0)
+        for k in parameters.keys():
+            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+
+    def _is_valid_requirements(self, requirements:Dict[str,Any])->None:
+        """Validation check for 'requirements' variable from main constructor. 
+        Called within parent BaseRecipe constructor."""
+        valid_dict(requirements, str, Any, strict=False, min_length=0)
+        for k in requirements.keys():
+            valid_string(k, VALID_VARIABLE_NAME_CHARS)
+
+
+class BashHandler(BaseHandler):
+    # Config option, above which debug messages are ignored
+    debug_level:int
+    # Where print messages are sent
+    _print_target:Any
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
+            print:Any=sys.stdout, logging:int=0, pause_time:int=5)->None:
+        """BashHandler Constructor. This creates jobs to be executed as 
+        bash scripts. This does not run as a continuous thread to 
+        handle execution, but is invoked according to a factory pattern using 
+        the handle function. Note that if this handler is given to a MeowRunner
+        object, the job_queue_dir will be overwridden by its"""
+        super().__init__(name=name, pause_time=pause_time)
+        self._is_valid_job_queue_dir(job_queue_dir)
+        self.job_queue_dir = job_queue_dir
+        self._print_target, self.debug_level = setup_debugging(print, logging)
+        print_debug(self._print_target, self.debug_level, 
+            "Created new BashHandler instance", DEBUG_INFO)
+
+    def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
+        """Function to determine given an event defintion, if this handler can 
+        process it or not. This handler accepts events from watchdog with 
+        Bash recipes"""
+        try:
+            valid_event(event)
+            msg = ""
+            if type(event[EVENT_RULE].recipe) != BashRecipe:
+                msg = "Recipe is not a BashRecipe. "
+            if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG:
+                msg += f"Event type is not {EVENT_TYPE_WATCHDOG}."
+            if msg:
+                return False, msg
+            else:
+                return True, ""
+        except Exception as e:
+            return False, str(e)
+
+    def _is_valid_job_queue_dir(self, job_queue_dir)->None:
+        """Validation check for 'job_queue_dir' variable from main 
+        constructor."""
+        valid_dir_path(job_queue_dir, must_exist=False)
+        if not os.path.exists(job_queue_dir):
+            make_dir(job_queue_dir)
+
+    def get_created_job_type(self)->str:
+        return JOB_TYPE_BASH
+    
+    def create_job_recipe_file(self, job_dir:str, event:Dict[str,Any], 
+            params_dict:Dict[str,Any])->str:
+        # parameterise recipe and write as executeable script
+        base_script = parameterize_bash_script(
+            event[EVENT_RULE].recipe.recipe, params_dict
+        )
+        base_file = os.path.join(job_dir, "recipe.sh")
+        write_file(lines_to_string(base_script), base_file)
+        os.chmod(base_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH )
+
+        return os.path.join("$(dirname $0)", "recipe.sh")

recipes/jupyter_notebook_recipe.py

@@ -8,25 +8,24 @@ Author(s): David Marchant
 import os
 import nbformat
 import sys
+import stat
 
 from typing import Any, Tuple, Dict
 
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.correctness.meow import valid_event
+from meow_base.core.meow import valid_event
-from core.correctness.validation import check_type, valid_string, \
+from meow_base.functionality.validation import check_type, valid_string, \
-    valid_dict, valid_path, valid_dir_path
+    valid_dict, valid_path, valid_dir_path, valid_existing_file_path
-from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
+from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
-    DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
+    DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
-    EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \
+    JOB_TYPE_PAPERMILL, EVENT_RULE, EVENT_TYPE, EVENT_RULE
-    JOB_ID, WATCHDOG_BASE, META_FILE, \
+from meow_base.functionality.debug import setup_debugging, print_debug
-    PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
+from meow_base.functionality.file_io import make_dir, read_notebook, \
-    EVENT_RULE, get_base_file
+    write_notebook
-from functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.parameterisation import \
-from functionality.file_io import make_dir, read_notebook, write_notebook, \
+    parameterize_jupyter_notebook
-    write_yaml
+from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG
-from functionality.meow import create_job, replace_keywords
-
 
 class JupyterNotebookRecipe(BaseRecipe):
     # A path to the jupyter notebook used to create this recipe
@@ -69,46 +68,20 @@ class PapermillHandler(BaseHandler):
     debug_level:int
     # Where print messages are sent
     _print_target:Any
-    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
-            print:Any=sys.stdout, logging:int=0)->None:
+            print:Any=sys.stdout, logging:int=0, pause_time:int=5)->None:
         """PapermillHandler Constructor. This creats jobs to be executed using 
         the papermill module. This does not run as a continuous thread to 
         handle execution, but is invoked according to a factory pattern using 
         the handle function. Note that if this handler is given to a MeowRunner
         object, the job_queue_dir will be overwridden."""
-        super().__init__()
+        super().__init__(name=name, pause_time=pause_time)
         self._is_valid_job_queue_dir(job_queue_dir)
         self.job_queue_dir = job_queue_dir
         self._print_target, self.debug_level = setup_debugging(print, logging)
         print_debug(self._print_target, self.debug_level, 
             "Created new PapermillHandler instance", DEBUG_INFO)
 
-    def handle(self, event:Dict[str,Any])->None:
-        """Function called to handle a given event."""
-        print_debug(self._print_target, self.debug_level, 
-            f"Handling event {event[EVENT_PATH]}", DEBUG_INFO)
-
-        rule = event[EVENT_RULE]
-
-        # Assemble job parameters dict from pattern variables
-        yaml_dict = {}
-        for var, val in rule.pattern.parameters.items():
-            yaml_dict[var] = val
-        for var, val in rule.pattern.outputs.items():
-            yaml_dict[var] = val
-        yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH]
-
-        # If no parameter sweeps, then one job will suffice
-        if not rule.pattern.sweep:
-            self.setup_job(event, yaml_dict)
-        else:
-            # If parameter sweeps, then many jobs created
-            values_list = rule.pattern.expand_sweeps()
-            for values in values_list:
-                for value in values:
-                    yaml_dict[value[0]] = value[1]
-                self.setup_job(event, yaml_dict)
-
     def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
         """Function to determine given an event defintion, if this handler can 
         process it or not. This handler accepts events from watchdog with 
@@ -134,123 +107,35 @@ class PapermillHandler(BaseHandler):
         if not os.path.exists(job_queue_dir):
             make_dir(job_queue_dir)
 
-    def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None:
+    def get_created_job_type(self)->str:
-        """Function to set up new job dict and send it to the runner to be 
+        return JOB_TYPE_PAPERMILL
-        executed."""
+    
-        meow_job = create_job(
+    def create_job_recipe_file(self, job_dir:str, event:Dict[str,Any], 
-            JOB_TYPE_PAPERMILL, 
+            params_dict:Dict[str,Any])->str: 
-            event, 
+        # parameterise recipe and write as executeable script
-            extras={
+        base_script = parameterize_jupyter_notebook(
-                JOB_PARAMETERS:yaml_dict,
+            event[EVENT_RULE].recipe.recipe, params_dict
-                JOB_HASH: event[WATCHDOG_HASH],
-                PYTHON_FUNC:papermill_job_func,
-            }
         )
-        print_debug(self._print_target, self.debug_level,  
+        base_file = os.path.join(job_dir, "recipe.ipynb")
-            f"Creating job from event at {event[EVENT_PATH]} of type "
-            f"{JOB_TYPE_PAPERMILL}.", DEBUG_INFO)
 
-        # replace MEOW keyworks within variables dict
+        write_notebook(base_script, base_file)
-        yaml_dict = replace_keywords(
+
-            meow_job[JOB_PARAMETERS],
+        os.chmod(base_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH )
-            meow_job[JOB_ID],
+
-            event[EVENT_PATH],
+        return f"papermill {base_file} {os.path.join(job_dir, 'result.ipynb')}"
-            event[WATCHDOG_BASE]
+
+def get_recipe_from_notebook(name:str, notebook_filename:str, 
+        parameters:Dict[str,Any]={}, requirements:Dict[str,Any]={}
+        )->JupyterNotebookRecipe:
+    valid_existing_file_path(notebook_filename, extension=".ipynb")
+    check_type(name, str, hint="get_recipe_from_notebook.name")
+
+    notebook_code = read_notebook(notebook_filename)
+
+    return JupyterNotebookRecipe(
+        name, 
+        notebook_code, 
+        parameters=parameters, 
+        requirements=requirements, 
+        source=notebook_filename
     )
-
-        # Create a base job directory
-        job_dir = os.path.join(self.job_queue_dir, meow_job[JOB_ID])
-        make_dir(job_dir)
-
-        # write a status file to the job directory
-        meta_file = os.path.join(job_dir, META_FILE)
-        write_yaml(meow_job, meta_file)
-
-        # write an executable notebook to the job directory
-        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
-        write_notebook(event[EVENT_RULE].recipe.recipe, base_file)
-
-        # write a parameter file to the job directory
-        param_file = os.path.join(job_dir, PARAMS_FILE)
-        write_yaml(yaml_dict, param_file)
-
-        meow_job[JOB_STATUS] = STATUS_QUEUED
-
-        # update the status file with queued status
-        write_yaml(meow_job, meta_file)
-        
-        # Send job directory, as actual definitons will be read from within it
-        self.to_runner.send(job_dir)
-
-# Papermill job execution code, to be run within the conductor
-def papermill_job_func(job_dir):
-    # Requires own imports as will be run in its own execution environment
-    import os
-    import papermill
-    from datetime import datetime
-    from core.correctness.vars import JOB_EVENT, JOB_ID, \
-        EVENT_PATH, META_FILE, PARAMS_FILE, \
-        JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
-        JOB_ERROR, STATUS_FAILED, get_job_file, \
-        get_result_file
-    from functionality.file_io import read_yaml, write_notebook, write_yaml
-    from functionality.hashing import get_file_hash
-    from functionality.parameterisation import parameterize_jupyter_notebook
-
-
-    # Identify job files
-    meta_file = os.path.join(job_dir, META_FILE)
-    base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
-    job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PAPERMILL))
-    result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL))
-    param_file = os.path.join(job_dir, PARAMS_FILE)
-
-    # Get job defintions   
-    job = read_yaml(meta_file)
-    yaml_dict = read_yaml(param_file)
-
-    # Check the hash of the triggering file, if present. This addresses 
-    # potential race condition as file could have been modified since 
-    # triggering event
-    if JOB_HASH in job:
-        # get current hash
-        triggerfile_hash = get_file_hash(job[JOB_EVENT][EVENT_PATH], SHA256)
-        # If hash doesn't match, then abort the job. If its been modified, then
-        # another job will have been scheduled anyway.
-        if not triggerfile_hash \
-                or triggerfile_hash != job[JOB_HASH]:
-            job[JOB_STATUS] = STATUS_SKIPPED
-            job[JOB_END_TIME] = datetime.now()
-            msg = "Job was skipped as triggering file " + \
-                f"'{job[JOB_EVENT][EVENT_PATH]}' has been modified since " + \
-                "scheduling. Was expected to have hash " + \
-                f"'{job[JOB_HASH]}' but has '{triggerfile_hash}'."
-            job[JOB_ERROR] = msg
-            write_yaml(job, meta_file)
-            return
-
-    # Create a parameterised version of the executable notebook
-    try:
-        base_notebook = read_notebook(base_file)
-        job_notebook = parameterize_jupyter_notebook(
-            base_notebook, yaml_dict
-        )
-        write_notebook(job_notebook, job_file)
-    except Exception as e:
-        job[JOB_STATUS] = STATUS_FAILED
-        job[JOB_END_TIME] = datetime.now()
-        msg = f"Job file {job[JOB_ID]} was not created successfully. {e}"
-        job[JOB_ERROR] = msg
-        write_yaml(job, meta_file)
-        return
-
-    # Execute the parameterised notebook
-    try:
-        papermill.execute_notebook(job_file, result_file, {})
-    except Exception as e:
-        job[JOB_STATUS] = STATUS_FAILED
-        job[JOB_END_TIME] = datetime.now()
-        msg = f"Result file {result_file} was not created successfully. {e}"
-        job[JOB_ERROR] = msg
-        write_yaml(job, meta_file)
-        return

recipes/python_recipe.py

@@ -6,26 +6,24 @@ along with an appropriate handler for said events.
 Author(s): David Marchant
 """
 import os
+import stat
 import sys
 
 from typing import Any, Tuple, Dict, List
 
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.correctness.meow import valid_event
+from meow_base.core.meow import valid_event
-from core.correctness.validation import check_script, valid_string, \
+from meow_base.functionality.validation import check_script, valid_string, \
     valid_dict, valid_dir_path
-from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
+from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
-    DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
+    DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, \
-    EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
+    JOB_TYPE_PYTHON, EVENT_TYPE, EVENT_RULE
-    JOB_ID, WATCHDOG_BASE, META_FILE, \
+from meow_base.functionality.debug import setup_debugging, print_debug
-    PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
+from meow_base.functionality.file_io import make_dir, write_file, \
-    get_base_file
+    lines_to_string
-from functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.parameterisation import parameterize_python_script
-from functionality.file_io import make_dir, read_file_lines, write_file, \
+from meow_base.patterns.file_event_pattern import EVENT_TYPE_WATCHDOG
-    write_yaml, lines_to_string
-from functionality.meow import create_job, replace_keywords
-
 
 class PythonRecipe(BaseRecipe):
     def __init__(self, name:str, recipe:List[str], parameters:Dict[str,Any]={}, 
@@ -59,45 +57,26 @@ class PythonHandler(BaseHandler):
     debug_level:int
     # Where print messages are sent
     _print_target:Any
-    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 
+    def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
-            print:Any=sys.stdout, logging:int=0)->None:
+            print:Any=sys.stdout, logging:int=0, pause_time:int=5)->None:
         """PythonHandler Constructor. This creates jobs to be executed as 
         python functions. This does not run as a continuous thread to 
         handle execution, but is invoked according to a factory pattern using 
         the handle function. Note that if this handler is given to a MeowRunner
-        object, the job_queue_dir will be overwridden but its"""
+        object, the job_queue_dir will be overwridden by its"""
-        super().__init__()
+        super().__init__(name=name, pause_time=pause_time)
         self._is_valid_job_queue_dir(job_queue_dir)
         self.job_queue_dir = job_queue_dir
         self._print_target, self.debug_level = setup_debugging(print, logging)
         print_debug(self._print_target, self.debug_level, 
             "Created new PythonHandler instance", DEBUG_INFO)
 
-    def handle(self, event:Dict[str,Any])->None:
+    def _is_valid_job_queue_dir(self, job_queue_dir)->None:
-        """Function called to handle a given event."""
+        """Validation check for 'job_queue_dir' variable from main 
-        print_debug(self._print_target, self.debug_level, 
+        constructor."""
-            f"Handling event {event[EVENT_PATH]}", DEBUG_INFO)
+        valid_dir_path(job_queue_dir, must_exist=False)
-
+        if not os.path.exists(job_queue_dir):
-        rule = event[EVENT_RULE]
+            make_dir(job_queue_dir)
-
-        # Assemble job parameters dict from pattern variables
-        yaml_dict = {}
-        for var, val in rule.pattern.parameters.items():
-            yaml_dict[var] = val
-        for var, val in rule.pattern.outputs.items():
-            yaml_dict[var] = val
-        yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH]
-
-        # If no parameter sweeps, then one job will suffice
-        if not rule.pattern.sweep:
-            self.setup_job(event, yaml_dict)
-        else:
-            # If parameter sweeps, then many jobs created
-            values_list = rule.pattern.expand_sweeps()
-            for values in values_list:
-                for value in values:
-                    yaml_dict[value[0]] = value[1]
-                self.setup_job(event, yaml_dict)
 
     def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
         """Function to determine given an event defintion, if this handler can 
@@ -108,8 +87,8 @@ class PythonHandler(BaseHandler):
             msg = ""
             if type(event[EVENT_RULE].recipe) != PythonRecipe:
                 msg = "Recipe is not a PythonRecipe. "
-            if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG:
+            # if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG:
-                msg += f"Event type is not {EVENT_TYPE_WATCHDOG}."
+            #     msg += f"Event type is not {EVENT_TYPE_WATCHDOG}."
             if msg:
                 return False, msg
             else:
@@ -117,151 +96,19 @@ class PythonHandler(BaseHandler):
         except Exception as e:
             return False, str(e)
 
-    def _is_valid_job_queue_dir(self, job_queue_dir)->None:
+    def get_created_job_type(self)->str:
-        """Validation check for 'job_queue_dir' variable from main 
+        return JOB_TYPE_PYTHON
-        constructor."""
-        valid_dir_path(job_queue_dir, must_exist=False)
-        if not os.path.exists(job_queue_dir):
-            make_dir(job_queue_dir)
 
-    def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None:
+    def create_job_recipe_file(self, job_dir:str, event:Dict[str,Any], 
-        """Function to set up new job dict and send it to the runner to be 
+            params_dict: Dict[str,Any])->str:
-        executed."""
+        # parameterise recipe and write as executeable script
-        meow_job = create_job(
+        base_script = parameterize_python_script(
-            JOB_TYPE_PYTHON, 
+            event[EVENT_RULE].recipe.recipe, params_dict
-            event, 
-            extras={
-                JOB_PARAMETERS:yaml_dict,
-                JOB_HASH: event[WATCHDOG_HASH],
-                PYTHON_FUNC:python_job_func
-            }
         )
-        print_debug(self._print_target, self.debug_level,  
+        base_file = os.path.join(job_dir, "recipe.py")
-            f"Creating job from event at {event[EVENT_PATH]} of type "
-            f"{JOB_TYPE_PYTHON}.", DEBUG_INFO)
 
-        # replace MEOW keyworks within variables dict
+        write_file(lines_to_string(base_script), base_file)
-        yaml_dict = replace_keywords(
+        os.chmod(base_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH )
-            meow_job[JOB_PARAMETERS],
-            meow_job[JOB_ID],
-            event[EVENT_PATH],
-            event[WATCHDOG_BASE]
-        )
 
-        # Create a base job directory
+        return f"python3 {base_file} >>{os.path.join(job_dir, 'output.log')} 2>&1"
-        job_dir = os.path.join(self.job_queue_dir, meow_job[JOB_ID])
-        make_dir(job_dir)
 
-        # write a status file to the job directory
-        meta_file = os.path.join(job_dir, META_FILE)
-        write_yaml(meow_job, meta_file)
-
-        # write an executable script to the job directory
-        base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
-        write_file(lines_to_string(event[EVENT_RULE].recipe.recipe), base_file)
-
-        # write a parameter file to the job directory
-        param_file = os.path.join(job_dir, PARAMS_FILE)
-        write_yaml(yaml_dict, param_file)
-
-        meow_job[JOB_STATUS] = STATUS_QUEUED
-
-        # update the status file with queued status
-        write_yaml(meow_job, meta_file)
-        
-        # Send job directory, as actual definitons will be read from within it
-        self.to_runner.send(job_dir)
-
-
-# Papermill job execution code, to be run within the conductor
-def python_job_func(job_dir):
-    # Requires own imports as will be run in its own execution environment
-    import sys
-    import os
-    from datetime import datetime
-    from io import StringIO
-    from core.correctness.vars import JOB_EVENT, JOB_ID, \
-        EVENT_PATH, META_FILE, PARAMS_FILE, \
-        JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
-        JOB_ERROR, STATUS_FAILED, get_base_file, \
-        get_job_file, get_result_file
-    from functionality.file_io import read_yaml, write_yaml
-    from functionality.hashing import get_file_hash
-    from functionality.parameterisation import parameterize_python_script
-
-    # Identify job files
-    meta_file = os.path.join(job_dir, META_FILE)
-    base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
-    job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_PYTHON))
-    result_file = os.path.join(job_dir, get_result_file(JOB_TYPE_PYTHON))
-    param_file = os.path.join(job_dir, PARAMS_FILE)
-
-    # Get job defintions   
-    job = read_yaml(meta_file)
-    yaml_dict = read_yaml(param_file)
-
-    # Check the hash of the triggering file, if present. This addresses 
-    # potential race condition as file could have been modified since 
-    # triggering event
-    if JOB_HASH in job:
-        # get current hash
-        triggerfile_hash = get_file_hash(job[JOB_EVENT][EVENT_PATH], SHA256)
-        # If hash doesn't match, then abort the job. If its been modified, then
-        # another job will have been scheduled anyway.
-        if not triggerfile_hash \
-                or triggerfile_hash != job[JOB_HASH]:
-            job[JOB_STATUS] = STATUS_SKIPPED
-            job[JOB_END_TIME] = datetime.now()
-            msg = "Job was skipped as triggering file " + \
-                f"'{job[JOB_EVENT][EVENT_PATH]}' has been modified since " + \
-                "scheduling. Was expected to have hash " + \
-                f"'{job[JOB_HASH]}' but has '{triggerfile_hash}'."
-            job[JOB_ERROR] = msg
-            write_yaml(job, meta_file)
-            return
-
-    # Create a parameterised version of the executable script
-    try:
-        base_script = read_file_lines(base_file)
-        job_script = parameterize_python_script(
-            base_script, yaml_dict
-        )
-        write_file(lines_to_string(job_script), job_file)
-    except Exception as e:
-        job[JOB_STATUS] = STATUS_FAILED
-        job[JOB_END_TIME] = datetime.now()
-        msg = f"Job file {job[JOB_ID]} was not created successfully. {e}"
-        job[JOB_ERROR] = msg
-        write_yaml(job, meta_file)
-        return
-
-    # Execute the parameterised script
-    std_stdout = sys.stdout
-    std_stderr = sys.stderr
-    try:
-        redirected_output = sys.stdout = StringIO()
-        redirected_error = sys.stderr = StringIO()
-
-        exec(open(job_file).read())        
-
-        write_file(("--STDOUT--\n"
-            f"{redirected_output.getvalue()}\n"
-            "\n"
-            "--STDERR--\n"
-            f"{redirected_error.getvalue()}\n"
-            ""), 
-            result_file)
-
-    except Exception as e:
-        sys.stdout = std_stdout
-        sys.stderr = std_stderr
-
-        job[JOB_STATUS] = STATUS_FAILED
-        job[JOB_END_TIME] = datetime.now()
-        msg = f"Result file {result_file} was not created successfully. {e}"
-        job[JOB_ERROR] = msg
-        write_yaml(job, meta_file)
-        return
-
-    sys.stdout = std_stdout
-    sys.stderr = std_stderr

rules/__init__.py

@@ -1,3 +0,0 @@
-
-from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
-from rules.file_event_python_rule import FileEventPythonRule

rules/file_event_jupyter_notebook_rule.py

@@ -1,43 +0,0 @@
-
-"""
-This file contains definitions for a MEOW rule connecting the FileEventPattern
-and JupyterNotebookRecipe.
-
-Author(s): David Marchant
-"""
-
-from core.base_rule import BaseRule
-from core.correctness.validation import check_type
-from patterns.file_event_pattern import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
-
-# TODO potentailly remove this and just invoke BaseRule directly, as does not
-# add any functionality other than some validation.
-class FileEventJupyterNotebookRule(BaseRule):
-    pattern_type = "FileEventPattern"
-    recipe_type = "JupyterNotebookRecipe" 
-    def __init__(self, name: str, pattern:FileEventPattern, 
-            recipe:JupyterNotebookRecipe):
-        super().__init__(name, pattern, recipe)
-        if pattern.recipe != recipe.name:
-            raise ValueError(f"Cannot create Rule {name}. Pattern "
-                f"{pattern.name} does not identify Recipe {recipe.name}. It "
-                f"uses {pattern.recipe}")
-
-    def _is_valid_pattern(self, pattern:FileEventPattern)->None:
-        """Validation check for 'pattern' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            pattern, 
-            FileEventPattern,
-            hint="FileEventJupyterNotebookRule.pattern"
-        )
-
-    def _is_valid_recipe(self, recipe:JupyterNotebookRecipe)->None:
-        """Validation check for 'recipe' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            recipe, 
-            JupyterNotebookRecipe,
-            hint="FileEventJupyterNotebookRule.recipe"
-        )

rules/file_event_python_rule.py

@@ -1,39 +0,0 @@
-
-"""
-This file contains definitions for a MEOW rule connecting the FileEventPattern
-and PythonRecipe.
-
-Author(s): David Marchant
-"""
-
-from core.base_rule import BaseRule
-from core.correctness.validation import check_type
-from patterns.file_event_pattern import FileEventPattern
-from recipes.python_recipe import PythonRecipe
-
-# TODO potentailly remove this and just invoke BaseRule directly, as does not
-# add any functionality other than some validation.
-class FileEventPythonRule(BaseRule):
-    pattern_type = "FileEventPattern"
-    recipe_type = "PythonRecipe" 
-    def __init__(self, name: str, pattern:FileEventPattern, 
-            recipe:PythonRecipe):
-        super().__init__(name, pattern, recipe)
-
-    def _is_valid_pattern(self, pattern:FileEventPattern)->None:
-        """Validation check for 'pattern' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            pattern, 
-            FileEventPattern,
-            hint="FileEventPythonRule.pattern"
-        )
-
-    def _is_valid_recipe(self, recipe:PythonRecipe)->None:
-        """Validation check for 'recipe' variable from main constructor. Is 
-        automatically called during initialisation."""
-        check_type(
-            recipe, 
-            PythonRecipe,
-            hint="FileEventPythonRule.recipe"
-        )

tests/shared.py

@@ -6,11 +6,13 @@ Author(s): David Marchant
 import os
 
 from distutils.dir_util import copy_tree
+from typing import List
 
-from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
+from meow_base.core.vars import DEFAULT_JOB_OUTPUT_DIR, \
-from functionality.file_io import make_dir, rmtree
+    DEFAULT_JOB_QUEUE_DIR, LOCK_EXT
-from patterns import FileEventPattern
+from meow_base.functionality.file_io import make_dir, rmtree
-from recipes import JupyterNotebookRecipe
+from meow_base.patterns.file_event_pattern import FileEventPattern
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
 
 # testing 
 TEST_DIR = "test_files"
@@ -35,44 +37,57 @@ def teardown():
     rmtree(DEFAULT_JOB_OUTPUT_DIR)
     rmtree(DEFAULT_JOB_QUEUE_DIR)
     rmtree("first")
-    if os.path.exists("temp_phantom_info.h5"):
+    for f in [
-        os.remove("temp_phantom_info.h5")
+                "temp_phantom_info.h5", 
-    if os.path.exists("temp_phantom.h5"):
+                "temp_phantom.h5", 
-        os.remove("temp_phantom.h5")
+                f"doesNotExist{LOCK_EXT}"
+            ]:
+        if os.path.exists(f):
+            os.remove(f)
 
 def backup_before_teardown(backup_source:str, backup_dest:str):
     make_dir(backup_dest, ensure_clean=True)
     copy_tree(backup_source, backup_dest)
 
+# Necessary, as the creation of locks is not deterministic
+def list_non_locks(dir:str)->List[str]:
+    return [f for f in os.listdir(dir) if not f.endswith(LOCK_EXT)]
 
-# Recipe funcs
+# Necessary, as the creation of locks is not deterministic
-BAREBONES_PYTHON_SCRIPT = [
+def count_non_locks(dir:str)->int:
+    return len(list_non_locks(dir))
+
+
+# Bash scripts
+BAREBONES_BASH_SCRIPT = [
     ""
 ]
-COMPLETE_PYTHON_SCRIPT = [
+COMPLETE_BASH_SCRIPT = [
-    "import os",
+    '#!/bin/bash',
-    "# Setup parameters",
+    '',
-    "num = 1000",
+    'num=1000',
-    "infile = 'somehere"+ os.path.sep +"particular'",
+    'infile="data"',
-    "outfile = 'nowhere"+ os.path.sep +"particular'",
+    'outfile="output"',
-    "",
+    '',
-    "with open(infile, 'r') as file:",
+    'echo "starting"',
-    "    s = float(file.read())",
+    '',
-    ""
+    'read var < $infile',
-    "for i in range(num):",
+    'for (( i=0; i<$num; i++ ))',
-    "    s += i",
+    'do',
-    "",
+    '    var=$((var+i))',
-    "div_by = 4",
+    'done',
-    "result = s / div_by",
+    '',
-    "",
+    'div_by=4',
-    "print(result)",
+    'echo $var',
-    "",
+    'result=$((var/div_by))',
-    "os.makedirs(os.path.dirname(outfile), exist_ok=True)",
+    '',
-    "",
+    'echo $result',
-    "with open(outfile, 'w') as file:",
+    '',
-    "    file.write(str(result))",
+    'mkdir -p $(dirname $outfile)',
-    "",
+    '',
-    "print('done')"
+    'echo $result > $outfile',
+    '',
+    'echo "done"'
 ]
 
 # Jupyter notebooks
@@ -1224,7 +1239,35 @@ GENERATOR_NOTEBOOK = {
 }
 
 # Python scripts
-IDMC_UTILS_MODULE = [
+BAREBONES_PYTHON_SCRIPT = [
+    ""
+]
+COMPLETE_PYTHON_SCRIPT = [
+    "import os",
+    "# Setup parameters",
+    "num = 1000",
+    "infile = 'somehere"+ os.path.sep +"particular'",
+    "outfile = 'nowhere"+ os.path.sep +"particular'",
+    "",
+    "with open(infile, 'r') as file:",
+    "    s = float(file.read())",
+    ""
+    "for i in range(num):",
+    "    s += i",
+    "",
+    "div_by = 4",
+    "result = s / div_by",
+    "",
+    "print(result)",
+    "",
+    "os.makedirs(os.path.dirname(outfile), exist_ok=True)",
+    "",
+    "with open(outfile, 'w') as file:",
+    "    file.write(str(result))",
+    "",
+    "print('done')"
+]
+IDMC_UTILS_PYTHON_SCRIPT = [
     "import matplotlib.pyplot as plt",
     "from sklearn import mixture",
     "import numpy as np",
@@ -1333,7 +1376,7 @@ IDMC_UTILS_MODULE = [
     "    else:",
     "        return means, stds, weights"
 ]
-GENERATE_SCRIPT = [
+GENERATE_PYTHON_SCRIPT = [
     "import numpy as np",
     "import random",
     "import foam_ct_phantom.foam_ct_phantom as foam_ct_phantom",
@@ -1360,7 +1403,13 @@ GENERATE_SCRIPT = [
     "    np.save(filename, dataset)",
     "    del dataset"
 ]
-
+COUNTING_PYTHON_SCRIPT = [
+    "import os",
+    "",
+    "dir_to_count = '.'",
+    "",
+    "print(f'There are {len(os.listdir(dir_to_count))} files in the directory.')"
+]
 
 valid_pattern_one = FileEventPattern(
     "pattern_one", "path_one", "recipe_one", "file_one")

tests/test_all.sh

@@ -7,15 +7,28 @@ starting_working_dir=$(pwd)
 script_name=$(basename "$0")
 script_dir=$(dirname "$(realpath "$0")")
 
+skip_long_tests=0
+
+for arg in "$@"
+do
+  if [[ $arg == -s ]]
+  then
+    skip_long_tests=1
+  fi
+done
+
 cd $script_dir
 
 # Gather all other test files and run pytest
 search_dir=.
 for entry in "$search_dir"/*
 do
-  if  [[ $entry == ./test* ]] && [[ -f $entry ]] && [[ $entry != ./$script_name ]] && [[ $entry != ./shared.py ]];
+  if  [[ $entry == ./test* ]] \
+    && [[ -f $entry ]] \
+    && [[ $entry != ./$script_name ]] \
+    && [[ $entry != ./shared.py ]];
   then
-    pytest $entry "-W ignore::DeprecationWarning"
+    SKIP_LONG=$skip_long_tests pytest $entry "-W ignore::DeprecationWarning"
   fi
 done
 

tests/test_base.py

@@ -1,17 +1,16 @@
 
 import unittest
  
-from typing import Any, Union, Tuple, Dict
+from typing import Any, Union, Tuple, Dict, List
 
-from core.base_conductor import BaseConductor
+from meow_base.core.base_conductor import BaseConductor
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.base_monitor import BaseMonitor
+from meow_base.core.base_monitor import BaseMonitor
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_rule import BaseRule
+from meow_base.core.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
-from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from patterns import FileEventPattern
+from shared import setup, teardown
-from shared import setup, teardown, valid_pattern_one, valid_recipe_one
 
 
 class BaseRecipeTests(unittest.TestCase):
@@ -147,35 +146,7 @@ class BasePatternTests(unittest.TestCase):
         self.assertEqual(len(values), 0)
 
 
-class BaseRuleTests(unittest.TestCase):
+# TODO test for base functions
-    def setUp(self)->None:
-        super().setUp()
-        setup()
-
-    def tearDown(self)->None:
-        super().tearDown()
-        teardown()
-
-    # Test that BaseRecipe instantiation
-    def testBaseRule(self)->None:
-        with self.assertRaises(TypeError):
-            BaseRule("name", "", "")
-
-        class NewRule(BaseRule):
-            pass
-        with self.assertRaises(NotImplementedError):
-            NewRule("name", "", "")
-
-        class FullRule(BaseRule):
-            pattern_type = "pattern"
-            recipe_type = "recipe"
-            def _is_valid_recipe(self, recipe:Any)->None:
-                pass
-            def _is_valid_pattern(self, pattern:Any)->None:
-                pass
-        FullRule("name", valid_pattern_one, valid_recipe_one)
-
-
 class BaseMonitorTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
@@ -201,32 +172,15 @@ class BaseMonitorTests(unittest.TestCase):
                 pass
             def stop(self):
                 pass
-            def _is_valid_patterns(self, patterns:Dict[str,BasePattern])->None:
+            def _get_valid_pattern_types(self)->List[type]:
-                pass
+                return [BasePattern]
-            def _is_valid_recipes(self, recipes:Dict[str,BaseRecipe])->None:
+            def _get_valid_recipe_types(self)->List[type]:
-                pass
+                return [BaseRecipe]
-            def add_pattern(self, pattern:BasePattern)->None:
-                pass
-            def update_pattern(self, pattern:BasePattern)->None:
-                pass
-            def remove_pattern(self, pattern:Union[str,BasePattern])->None:
-                pass
-            def get_patterns(self)->None:
-                pass
-            def add_recipe(self, recipe:BaseRecipe)->None:
-                pass
-            def update_recipe(self, recipe:BaseRecipe)->None:
-                pass
-            def remove_recipe(self, recipe:Union[str,BaseRecipe])->None:
-                pass
-            def get_recipes(self)->None:
-                pass
-            def get_rules(self)->None:
-                pass
             
         FullTestMonitor({}, {})
 
 
+# TODO test for base functions
 class BaseHandleTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
@@ -248,21 +202,19 @@ class BaseHandleTests(unittest.TestCase):
             TestHandler()
 
         class FullTestHandler(BaseHandler):
-            def handle(self, event):
-                pass
-            def start(self):
-                pass
-            def stop(self):
-                pass
-            def _is_valid_inputs(self, inputs:Any)->None:
-                pass
             def valid_handle_criteria(self, event:Dict[str,Any]
                     )->Tuple[bool,str]:
                 pass
+            def get_created_job_type(self)->str:
+                pass
+            def create_job_recipe_file(self, job_dir:str, event:Dict[str,Any], 
+                    params_dict:Dict[str,Any])->str:
+                pass
 
         FullTestHandler()
 
 
+# TODO test for base functions
 class BaseConductorTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
@@ -284,9 +236,6 @@ class BaseConductorTests(unittest.TestCase):
             TestConductor()
 
         class FullTestConductor(BaseConductor):
-            def execute(self, job_dir:str)->None:
-                pass
-
             def valid_execute_criteria(self, job:Dict[str,Any]
                     )->Tuple[bool,str]:
                 pass

tests/test_functionality.py

@@ -8,40 +8,40 @@ from datetime import datetime
 from multiprocessing import Pipe, Queue
 from os.path import basename
 from sys import prefix, base_prefix
-from time import sleep
+from time import sleep, time
 from typing import Dict
 
-from core.base_rule import BaseRule
+from meow_base.core.meow import EVENT_KEYS
-from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
+from meow_base.core.rule import Rule
-    SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
+from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
-    WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
+    SHA256, EVENT_TYPE, EVENT_PATH, LOCK_EXT, EVENT_RULE, JOB_PARAMETERS, \
-    PYTHON_FUNC, JOB_ID, JOB_EVENT, \
+    PYTHON_FUNC, JOB_ID, JOB_EVENT, JOB_ERROR, STATUS_DONE, \
     JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
-    JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
+    JOB_REQUIREMENTS, JOB_TYPE_PAPERMILL, STATUS_CREATING
-from functionality.debug import setup_debugging
+from meow_base.functionality.debug import setup_debugging
-from functionality.file_io import lines_to_string, make_dir, read_file, \
+from meow_base.functionality.file_io import lines_to_string, make_dir, \
-    read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
+    read_file, read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
-    write_notebook, write_yaml    
+    write_notebook, write_yaml, threadsafe_read_status, \
-from functionality.hashing import get_file_hash
+    threadsafe_update_status, threadsafe_write_status
-from functionality.meow import create_event, create_job, create_rule, \
+from meow_base.functionality.hashing import get_hash
-    create_rules, create_watchdog_event, replace_keywords, \
+from meow_base.functionality.meow import KEYWORD_BASE, KEYWORD_DIR, \
-    create_parameter_sweep, \
+    KEYWORD_EXTENSION, KEYWORD_FILENAME, KEYWORD_JOB, KEYWORD_PATH, \
-    KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
+    KEYWORD_PREFIX, KEYWORD_REL_DIR, KEYWORD_REL_PATH, \
-    KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
+    create_event, create_job_metadata_dict, create_rule, create_rules, \
-    KEYWORD_REL_PATH
+    replace_keywords, create_parameter_sweep
-from functionality.naming import _generate_id
+from meow_base.functionality.naming import _generate_id
-from functionality.parameterisation import parameterize_jupyter_notebook, \
+from meow_base.functionality.parameterisation import \
-    parameterize_python_script
+    parameterize_jupyter_notebook, parameterize_python_script
-from functionality.process_io import wait
+from meow_base.functionality.process_io import wait
-from functionality.requirements import create_python_requirements, \
+from meow_base.functionality.requirements import REQUIREMENT_PYTHON, \
-    check_requirements, \
+    REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, REQ_PYTHON_VERSION, \
-    REQUIREMENT_PYTHON, REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, \
+    create_python_requirements, check_requirements
-    REQ_PYTHON_VERSION
+from meow_base.patterns.file_event_pattern import FileEventPattern, \
-from patterns import FileEventPattern
+    EVENT_TYPE_WATCHDOG, WATCHDOG_BASE, WATCHDOG_HASH
-from recipes import JupyterNotebookRecipe
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
-from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
+from shared import TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, \
-    valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
+    COMPLETE_PYTHON_SCRIPT, valid_recipe_two, valid_recipe_one, \
-    COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
+    valid_pattern_one, valid_pattern_two, setup, teardown
 
 class DebugTests(unittest.TestCase):
     def setUp(self)->None:
@@ -332,6 +332,243 @@ data"""
 
         self.assertEqual(lines_to_string(l), "a\nb\nc")
 
+    def testThreadsafeWriteStatus(self)->None:
+        first_yaml_dict = {
+            "A": "a",
+            "B": 1,
+            "C": {
+                "D": True,
+                "E": [
+                    1, 2, 3
+                ]
+            }
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+
+        self.assertFalse(os.path.exists(filepath))
+        threadsafe_write_status(first_yaml_dict, filepath)
+        self.assertTrue(os.path.exists(filepath))
+        self.assertTrue(os.path.exists(filepath + LOCK_EXT))
+
+        with open(filepath, 'r') as f:
+            data = f.readlines()
+        
+        expected_bytes = [
+            'A: a\n', 
+            'B: 1\n',
+            'C:\n', 
+            '  D: true\n',
+            '  E:\n',
+            '  - 1\n',
+            '  - 2\n',
+            '  - 3\n'
+        ]
+        
+        self.assertEqual(data, expected_bytes)
+
+        second_yaml_dict = {
+            "F": "a",
+            "G": 1,
+            "H": {
+                "I": True,
+                "J": [
+                    1, 2, 3
+                ]
+            }
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+
+        threadsafe_write_status(second_yaml_dict, filepath)
+        self.assertTrue(os.path.exists(filepath))
+        self.assertTrue(os.path.exists(filepath + LOCK_EXT))
+
+        with open(filepath, 'r') as f:
+            data = f.readlines()
+        
+        expected_bytes = [
+            'F: a\n', 
+            'G: 1\n',
+            'H:\n', 
+            '  I: true\n',
+            '  J:\n',
+            '  - 1\n',
+            '  - 2\n',
+            '  - 3\n'
+        ]
+        
+        self.assertEqual(data, expected_bytes)
+
+    def testThreadsafeReadStatus(self)->None:
+        yaml_dict = {
+            "A": "a",
+            "B": 1,
+            "C": {
+                "D": True,
+                "E": [
+                    1, 2, 3
+                ]
+            }
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+        threadsafe_write_status(yaml_dict, filepath)
+
+        read_dict = threadsafe_read_status(filepath)
+        self.assertEqual(yaml_dict, read_dict)
+
+        with self.assertRaises(FileNotFoundError):
+            threadsafe_read_status("doesNotExist")
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "T.txt")
+        with open(filepath, "w") as f:
+            f.write("Data")
+
+        data = threadsafe_read_status(filepath)
+        self.assertEqual(data, "Data")
+
+    def testThreadsafeUpdateStatus(self)->None:
+        first_yaml_dict = {
+            "A": "a",
+            "B": 1,
+            "C": {
+                "D": True,
+                "E": [
+                    1, 2, 3
+                ]
+            }
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+
+        self.assertFalse(os.path.exists(filepath))
+        threadsafe_write_status(first_yaml_dict, filepath)
+        self.assertTrue(os.path.exists(filepath))
+        self.assertTrue(os.path.exists(filepath + LOCK_EXT))
+
+        with open(filepath, 'r') as f:
+            data = f.readlines()
+        
+        expected_bytes = [
+            'A: a\n', 
+            'B: 1\n',
+            'C:\n', 
+            '  D: true\n',
+            '  E:\n',
+            '  - 1\n',
+            '  - 2\n',
+            '  - 3\n'
+        ]
+        
+        self.assertEqual(data, expected_bytes)
+
+        second_yaml_dict = {
+            "B": 42,
+            "C": {
+                "E": [
+                    1, 2, 3, 4
+                ]
+            }
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+
+        threadsafe_update_status(second_yaml_dict, filepath)
+        self.assertTrue(os.path.exists(filepath))
+        self.assertTrue(os.path.exists(filepath + LOCK_EXT))
+
+        with open(filepath, 'r') as f:
+            data = f.readlines()
+        
+        expected_bytes = [
+            'A: a\n', 
+            'B: 42\n',
+            'C:\n', 
+            '  E:\n',
+            '  - 1\n',
+            '  - 2\n',
+            '  - 3\n',
+            '  - 4\n'
+        ]
+        
+        self.assertEqual(data, expected_bytes)
+
+    def testThreadsafeUpdateProctectedStatus(self)->None:
+        first_yaml_dict = {
+            JOB_CREATE_TIME: "now",
+            JOB_STATUS: "Wham",
+            JOB_ERROR: "first error.",
+            JOB_ID: "id",
+            JOB_TYPE: "type"
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+
+        self.assertFalse(os.path.exists(filepath))
+        threadsafe_write_status(first_yaml_dict, filepath)
+        self.assertTrue(os.path.exists(filepath))
+        self.assertTrue(os.path.exists(filepath + LOCK_EXT))
+
+        status = threadsafe_read_status(filepath)
+        
+        self.assertEqual(first_yaml_dict, status)
+
+        second_yaml_dict = {
+            JOB_CREATE_TIME: "changed",
+            JOB_STATUS: STATUS_DONE,
+            JOB_ERROR: "changed.",
+            JOB_ID: "changed"
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+
+        threadsafe_update_status(second_yaml_dict, filepath)
+        self.assertTrue(os.path.exists(filepath))
+        self.assertTrue(os.path.exists(filepath + LOCK_EXT))
+
+        status = threadsafe_read_status(filepath)
+
+        expected_second_yaml_dict = {
+            JOB_CREATE_TIME: "now",
+            JOB_STATUS: STATUS_DONE,
+            JOB_ERROR: "first error. changed.",
+            JOB_ID: "changed",
+            JOB_TYPE: "type"
+        }
+        
+        self.assertEqual(expected_second_yaml_dict, status)
+
+        third_yaml_dict = {
+            JOB_CREATE_TIME: "editted",
+            JOB_STATUS: "editted",
+            JOB_ERROR: "editted.",
+            JOB_ID: "editted",
+            "something_new": "new"
+        }
+
+        filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
+
+        threadsafe_update_status(third_yaml_dict, filepath)
+        self.assertTrue(os.path.exists(filepath))
+        self.assertTrue(os.path.exists(filepath + LOCK_EXT))
+
+        status = threadsafe_read_status(filepath)
+
+        expected_third_yaml_dict = {
+            JOB_CREATE_TIME: "now",
+            JOB_STATUS: STATUS_DONE,
+            JOB_ERROR: "first error. changed. editted.",
+            JOB_ID: "editted",
+            JOB_TYPE: "type",
+            "something_new": "new"
+        }
+
+        print(expected_third_yaml_dict)
+        print(status)
+        
+        self.assertEqual(expected_third_yaml_dict, status)
+
 
 class HashingTests(unittest.TestCase):
     def setUp(self)->None:
@@ -342,7 +579,7 @@ class HashingTests(unittest.TestCase):
         super().tearDown()
         teardown()
 
-    # Test that get_file_hash produces the expected hash
+    # Test that get_hash produces the expected hash
     def testGetFileHashSha256(self)->None:
         file_path = os.path.join(TEST_MONITOR_BASE, "hased_file.txt")
         with open(file_path, 'w') as hashed_file:
@@ -350,15 +587,15 @@ class HashingTests(unittest.TestCase):
         expected_hash = \
             "8557122088c994ba8aa5540ccbb9a3d2d8ae2887046c2db23d65f40ae63abade"
         
-        hash = get_file_hash(file_path, SHA256)
+        hash = get_hash(file_path, SHA256)
         self.assertEqual(hash, expected_hash)
     
-    # Test that get_file_hash raises on a missing file
+    # Test that get_hash raises on a missing file
     def testGetFileHashSha256NoFile(self)->None:
         file_path = os.path.join(TEST_MONITOR_BASE, "file.txt")
 
         with self.assertRaises(FileNotFoundError):        
-            get_file_hash(file_path, SHA256)
+            get_hash(file_path, SHA256)
 
 
 class MeowTests(unittest.TestCase):
@@ -386,30 +623,32 @@ class MeowTests(unittest.TestCase):
 
         rule = create_rule(pattern, recipe)
 
-        event = create_event("test", "path", rule)
+        event = create_event("test", "path", rule, time())
 
         self.assertEqual(type(event), dict)
-        self.assertEqual(len(event.keys()), 3)
+        self.assertEqual(len(event.keys()), len(EVENT_KEYS))
-        self.assertTrue(EVENT_TYPE in event.keys())
+        for key, value in EVENT_KEYS.items():
-        self.assertTrue(EVENT_PATH in event.keys())
+            self.assertTrue(key in event.keys())
-        self.assertTrue(EVENT_RULE in event.keys())
+            self.assertIsInstance(event[key], value)
         self.assertEqual(event[EVENT_TYPE], "test")
         self.assertEqual(event[EVENT_PATH], "path")
         self.assertEqual(event[EVENT_RULE], rule)
 
-        event2 = create_event("test2", "path2", rule, extras={"a":1})
+        event2 = create_event(
+            "test2", "path2", rule, time(), extras={"a":1}
+        )
 
         self.assertEqual(type(event2), dict)
-        self.assertTrue(EVENT_TYPE in event2.keys())
+        self.assertEqual(len(event.keys()), len(EVENT_KEYS))
-        self.assertTrue(EVENT_PATH in event.keys())
+        for key, value in EVENT_KEYS.items():
-        self.assertTrue(EVENT_RULE in event.keys())
+            self.assertTrue(key in event.keys())
-        self.assertEqual(len(event2.keys()), 4)
+            self.assertIsInstance(event[key], value)
         self.assertEqual(event2[EVENT_TYPE], "test2")
         self.assertEqual(event2[EVENT_PATH], "path2")
         self.assertEqual(event2[EVENT_RULE], rule)
         self.assertEqual(event2["a"], 1)
 
-    # Test that create_job produces valid job dictionary
+    # Test that create_job_metadata_dict produces valid job dictionary
     def testCreateJob(self)->None:
         pattern = FileEventPattern(
             "pattern", 
@@ -429,6 +668,7 @@ class MeowTests(unittest.TestCase):
             EVENT_TYPE_WATCHDOG,
             "file_path",
             rule,
+            time(),
             extras={
                 WATCHDOG_BASE: TEST_MONITOR_BASE,
                 EVENT_RULE: rule,
@@ -436,7 +676,7 @@ class MeowTests(unittest.TestCase):
             }
         )
 
-        job_dict = create_job(
+        job_dict = create_job_metadata_dict(
             JOB_TYPE_PAPERMILL,
             event,
             extras={
@@ -445,7 +685,6 @@ class MeowTests(unittest.TestCase):
                     "infile":"file_path",
                     "outfile":"result_path"
                 },
-                JOB_HASH: "file_hash",
                 PYTHON_FUNC:max
             }
         )
@@ -464,64 +703,12 @@ class MeowTests(unittest.TestCase):
         self.assertIn(JOB_RULE, job_dict)
         self.assertEqual(job_dict[JOB_RULE], rule.name)
         self.assertIn(JOB_STATUS, job_dict)
-        self.assertEqual(job_dict[JOB_STATUS], STATUS_QUEUED)
+        self.assertEqual(job_dict[JOB_STATUS], STATUS_CREATING)
         self.assertIn(JOB_CREATE_TIME, job_dict)
         self.assertIsInstance(job_dict[JOB_CREATE_TIME], datetime)
         self.assertIn(JOB_REQUIREMENTS, job_dict)
         self.assertEqual(job_dict[JOB_REQUIREMENTS], {})
 
-    # Test creation of watchdog event dict
-    def testCreateWatchdogEvent(self)->None:
-        pattern = FileEventPattern(
-            "pattern", 
-            "file_path", 
-            "recipe_one", 
-            "infile", 
-            parameters={
-                "extra":"A line from a test Pattern",
-                "outfile":"result_path"
-            })
-        recipe = JupyterNotebookRecipe(
-            "recipe_one", APPENDING_NOTEBOOK)
-
-        rule = create_rule(pattern, recipe)
-
-        with self.assertRaises(TypeError):
-            event = create_watchdog_event("path", rule)
-
-        event = create_watchdog_event("path", rule, "base", "hash")
-
-        self.assertEqual(type(event), dict)
-        self.assertEqual(len(event.keys()), 5)
-        self.assertTrue(EVENT_TYPE in event.keys())
-        self.assertTrue(EVENT_PATH in event.keys())
-        self.assertTrue(EVENT_RULE in event.keys())
-        self.assertTrue(WATCHDOG_BASE in event.keys())
-        self.assertTrue(WATCHDOG_HASH in event.keys())
-        self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
-        self.assertEqual(event[EVENT_PATH], "path")
-        self.assertEqual(event[EVENT_RULE], rule)
-        self.assertEqual(event[WATCHDOG_BASE], "base")
-        self.assertEqual(event[WATCHDOG_HASH], "hash")
-
-        event = create_watchdog_event(
-            "path2", rule, "base", "hash", extras={"a":1}
-        )
-
-        self.assertEqual(type(event), dict)
-        self.assertTrue(EVENT_TYPE in event.keys())
-        self.assertTrue(EVENT_PATH in event.keys())
-        self.assertTrue(EVENT_RULE in event.keys())
-        self.assertTrue(WATCHDOG_BASE in event.keys())
-        self.assertTrue(WATCHDOG_HASH in event.keys())
-        self.assertEqual(len(event.keys()), 6)
-        self.assertEqual(event[EVENT_TYPE], EVENT_TYPE_WATCHDOG)
-        self.assertEqual(event[EVENT_PATH], "path2")
-        self.assertEqual(event[EVENT_RULE], rule)
-        self.assertEqual(event["a"], 1)
-        self.assertEqual(event[WATCHDOG_BASE], "base")
-        self.assertEqual(event[WATCHDOG_HASH], "hash")
-
     # Test that replace_keywords replaces MEOW keywords in a given dictionary
     def testReplaceKeywords(self)->None:
         test_dict = {
@@ -583,7 +770,7 @@ class MeowTests(unittest.TestCase):
     def testCreateRule(self)->None:
         rule = create_rule(valid_pattern_one, valid_recipe_one)
 
-        self.assertIsInstance(rule, BaseRule)
+        self.assertIsInstance(rule, Rule)
 
         with self.assertRaises(ValueError):
             rule = create_rule(valid_pattern_one, valid_recipe_two)
@@ -594,7 +781,7 @@ class MeowTests(unittest.TestCase):
 
         self.assertEqual(len(rules), 0)
 
-    # Test that create_rules creates rules from patterns and recipes
+    # Test that create_rules creates rules from meow_base.patterns and recipes
     def testCreateRulesPatternsAndRecipesDicts(self)->None:
         patterns = {
             valid_pattern_one.name: valid_pattern_one,
@@ -609,7 +796,7 @@ class MeowTests(unittest.TestCase):
         self.assertEqual(len(rules), 2)
         for k, rule in rules.items():
             self.assertIsInstance(k, str)
-            self.assertIsInstance(rule, BaseRule)
+            self.assertIsInstance(rule, Rule)
             self.assertEqual(k, rule.name)
 
     # Test that create_rules creates nothing from invalid pattern inputs
@@ -893,6 +1080,7 @@ class ProcessIoTests(unittest.TestCase):
                 msg = readable.recv()
                 self.assertEqual(msg, 1)
 
+
 class RequirementsTest(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()

tests/test_rule.py

@@ -0,0 +1,92 @@
+
+import unittest
+
+from meow_base.core.rule import Rule
+from meow_base.patterns.file_event_pattern import FileEventPattern
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
+from shared import BAREBONES_NOTEBOOK, setup, teardown
+
+class CorrectnessTests(unittest.TestCase):
+    def setUp(self)->None:
+        super().setUp()
+        setup()
+
+    def tearDown(self)->None:
+        super().tearDown()
+        teardown()
+        
+    # Test Rule created from valid pattern and recipe
+    def testRuleCreationMinimum(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        Rule(fep, jnr)
+
+    # Test Rule not created with empty name
+    def testRuleCreationNoName(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        r = Rule(fep, jnr)
+
+        self.assertIsInstance(r.name, str)
+        self.assertTrue(len(r.name) > 1)
+
+    # Test Rule not created with invalid name
+    def testRuleCreationInvalidName(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        with self.assertRaises(TypeError):
+            Rule(fep, jnr, name=1)
+
+    # Test Rule not created with invalid pattern
+    def testRuleCreationInvalidPattern(self)->None:
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        with self.assertRaises(TypeError):
+            Rule("pattern", jnr)
+
+    # Test Rule not created with invalid recipe
+    def testRuleCreationInvalidRecipe(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+
+        with self.assertRaises(TypeError):
+            Rule(fep, "recipe")
+
+    # Test Rule not created with mismatched recipe
+    def testRuleCreationMissmatchedRecipe(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
+
+        with self.assertRaises(ValueError):
+            Rule(fep, jnr)
+
+    # Test Rule created with valid name
+    def testRuleSetupName(self)->None:
+        name = "name"
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        fejnr = Rule(fep, jnr, name=name)
+
+        self.assertEqual(fejnr.name, name)
+
+    # Test Rule not created with valid pattern
+    def testRuleSetupPattern(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        fejnr = Rule(fep, jnr)
+
+        self.assertEqual(fejnr.pattern, fep)
+
+    # Test Rule not created with valid recipe
+    def testRuleSetupRecipe(self)->None:
+        fep = FileEventPattern("name", "path", "recipe", "file")
+        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
+
+        fejnr = Rule(fep, jnr)
+
+        self.assertEqual(fejnr.recipe, jnr)
+

tests/test_rules.py

@@ -1,90 +0,0 @@
-
-import unittest
-
-from patterns.file_event_pattern import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
-from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
-from shared import setup, teardown, BAREBONES_NOTEBOOK
-
-class CorrectnessTests(unittest.TestCase):
-    def setUp(self)->None:
-        super().setUp()
-        setup()
-
-    def tearDown(self)->None:
-        super().tearDown()
-        teardown()
-        
-    # Test FileEventJupyterNotebookRule created from valid pattern and recipe
-    def testFileEventJupyterNotebookRuleCreationMinimum(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        FileEventJupyterNotebookRule("name", fep, jnr)
-
-    # Test FileEventJupyterNotebookRule not created with empty name
-    def testFileEventJupyterNotebookRuleCreationNoName(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(ValueError):
-            FileEventJupyterNotebookRule("", fep, jnr)
-
-    # Test FileEventJupyterNotebookRule not created with invalid name
-    def testFileEventJupyterNotebookRuleCreationInvalidName(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(TypeError):
-            FileEventJupyterNotebookRule(1, fep, jnr)
-
-    # Test FileEventJupyterNotebookRule not created with invalid pattern
-    def testFileEventJupyterNotebookRuleCreationInvalidPattern(self)->None:
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(TypeError):
-            FileEventJupyterNotebookRule("name", "pattern", jnr)
-
-    # Test FileEventJupyterNotebookRule not created with invalid recipe
-    def testFileEventJupyterNotebookRuleCreationInvalidRecipe(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-
-        with self.assertRaises(TypeError):
-            FileEventJupyterNotebookRule("name", fep, "recipe")
-
-    # Test FileEventJupyterNotebookRule not created with mismatched recipe
-    def testFileEventJupyterNotebookRuleCreationMissmatchedRecipe(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
-
-        with self.assertRaises(ValueError):
-            FileEventJupyterNotebookRule("name", fep, jnr)
-
-    # Test FileEventJupyterNotebookRule created with valid name
-    def testFileEventJupyterNotebookRuleSetupName(self)->None:
-        name = "name"
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        fejnr = FileEventJupyterNotebookRule(name, fep, jnr)
-
-        self.assertEqual(fejnr.name, name)
-
-    # Test FileEventJupyterNotebookRule not created with valid pattern
-    def testFileEventJupyterNotebookRuleSetupPattern(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        fejnr = FileEventJupyterNotebookRule("name", fep, jnr)
-
-        self.assertEqual(fejnr.pattern, fep)
-
-    # Test FileEventJupyterNotebookRule not created with valid recipe
-    def testFileEventJupyterNotebookRuleSetupRecipe(self)->None:
-        fep = FileEventPattern("name", "path", "recipe", "file")
-        jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
-
-        fejnr = FileEventJupyterNotebookRule("name", fep, jnr)
-
-        self.assertEqual(fejnr.recipe, jnr)
-

tests/test_validation.py

@@ -3,20 +3,23 @@ import unittest
 import os
 
 from datetime import datetime
+from time import time
 from typing import Any, Union
 
-from core.correctness.meow import valid_event, valid_job, valid_watchdog_event
+from meow_base.core.meow import valid_event, valid_job
-from core.correctness.validation import check_type, check_implementation, \
+from meow_base.functionality.validation import check_type, \
-    valid_string, valid_dict, valid_list, valid_existing_file_path, \
+    check_implementation, valid_string, valid_dict, valid_list, \
-    valid_dir_path, valid_non_existing_path, check_callable
+    valid_existing_file_path, valid_dir_path, valid_non_existing_path, \
-from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
+    check_callable, valid_natural, valid_dict_multiple_types
-    EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
+from meow_base.core.vars import VALID_NAME_CHARS, SHA256, \
-    JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
+    EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
-    WATCHDOG_HASH
+    JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, EVENT_TIME
-from functionality.file_io import make_dir
+from meow_base.functionality.file_io import make_dir
-from functionality.meow import create_rule
+from meow_base.functionality.meow import create_rule
-from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \
+from meow_base.patterns.file_event_pattern import WATCHDOG_BASE, \
-    valid_recipe_one
+    WATCHDOG_HASH, valid_watchdog_event
+from shared import TEST_MONITOR_BASE, valid_pattern_one, valid_recipe_one, \
+    setup, teardown
 
 class ValidationTests(unittest.TestCase):
     def setUp(self)->None:
@@ -125,6 +128,36 @@ class ValidationTests(unittest.TestCase):
         with self.assertRaises(ValueError):
             valid_dict({"a": 0, "b": 1}, str, int, strict=True)
 
+    def testValidDictMultipleTypes(self)->None:
+        valid_dict_multiple_types(
+            {"a": 0, "b": 1}, 
+            str, 
+            [int],
+            strict=False
+        )
+
+        valid_dict_multiple_types(
+            {"a": 0, "b": 1}, 
+            str, 
+            [int, str],
+            strict=False
+        )
+        
+        valid_dict_multiple_types(
+            {"a": 0, "b": 'a'}, 
+            str, 
+            [int, str],
+            strict=False
+        )
+
+        with self.assertRaises(TypeError):
+            valid_dict_multiple_types(
+                {"a": 0, "b": 'a'}, 
+                str, 
+                [int],
+                strict=False
+            )
+
     # Test valid_list with sufficent lengths
     def testValidListMinimum(self)->None:
         valid_list([1, 2, 3], int)
@@ -253,6 +286,18 @@ class ValidationTests(unittest.TestCase):
         with self.assertRaises(TypeError):
             check_callable("a")
 
+    # Test natural number check
+    def testValidNatural(self)->None:
+        valid_natural(0)
+        valid_natural(1)
+
+        with self.assertRaises(ValueError):
+            valid_natural(-1)
+
+        with self.assertRaises(TypeError):
+            valid_natural(1.0)
+
+
 class MeowTests(unittest.TestCase):
     def setUp(self)->None:
         super().setUp()
@@ -269,12 +314,14 @@ class MeowTests(unittest.TestCase):
         valid_event({
             EVENT_TYPE: "test", 
             EVENT_PATH: "path", 
-            EVENT_RULE: rule
+            EVENT_RULE: rule,
+            EVENT_TIME: time()
         })
         valid_event({
             EVENT_TYPE: "anything", 
             EVENT_PATH: "path", 
             EVENT_RULE: rule,
+            EVENT_TIME: time(),
             "a": 1
         })
 
@@ -317,6 +364,7 @@ class MeowTests(unittest.TestCase):
             EVENT_TYPE: "test", 
             EVENT_PATH: "path", 
             EVENT_RULE: rule,
+            EVENT_TIME: time(),
             WATCHDOG_HASH: "hash",
             WATCHDOG_BASE: "base"
         })
@@ -325,7 +373,8 @@ class MeowTests(unittest.TestCase):
             valid_watchdog_event({
                 EVENT_TYPE: "test", 
                 EVENT_PATH: "path", 
-                EVENT_RULE: "rule"
+                EVENT_RULE: "rule",
+                EVENT_TIME: time()
             })
 
         with self.assertRaises(KeyError):
@@ -333,6 +382,7 @@ class MeowTests(unittest.TestCase):
                 EVENT_TYPE: "anything", 
                 EVENT_PATH: "path", 
                 EVENT_RULE: "rule",
+                EVENT_TIME: time(),
                 "a": 1
             })