updated runner structure so that handlers and conductors actually pull from queues in the runner. changes to logic in both are extensive, but most individual functinos are unaffected. I've also moved several functions that were part of individual monitor, handler and conductors to the base classes.

core/base_conductor.py

@@ -6,12 +6,14 @@ from for all conductor instances.
 Author(s): David Marchant
 """
 
-from typing import Any, Tuple, Dict
+from threading import Event, Thread
+from time import sleep
+from typing import Any, Tuple, Dict, Union
 
 from meow_base.core.vars import VALID_CONDUCTOR_NAME_CHARS, VALID_CHANNELS, \
     get_drt_imp_msg
 from meow_base.functionality.validation import check_implementation, \
-    valid_string
+    valid_string, valid_existing_dir_path, valid_natural
 from meow_base.functionality.naming import generate_conductor_id
 
 
@@ -19,10 +21,11 @@ class BaseConductor:
     # An identifier for a conductor within the runner. Can be manually set in 
     # the constructor, or autogenerated if no name provided.
     name:str
-    # A channel for sending messages to the runner. Note that this will be 
-    # overridden by a MeowRunner, if a conductor instance is passed to it, and 
-    # so does not need to be initialised within the conductor itself.
-    to_runner: VALID_CHANNELS
+    # A channel for sending messages to the runner job queue. Note that this 
+    # will be overridden by a MeowRunner, if a conductor instance is passed to 
+    # it, and so does not need to be initialised within the conductor itself, 
+    # unless the conductor is running independently of a runner.
+    to_runner_job: VALID_CHANNELS
     # Directory where queued jobs are initially written to. Note that this 
     # will be overridden by a MeowRunner, if a handler instance is passed to 
     # it, and so does not need to be initialised within the handler itself.
@@ -31,16 +34,20 @@ class BaseConductor:
     # will be overridden by a MeowRunner, if a handler instance is passed to 
     # it, and so does not need to be initialised within the handler itself.
     job_output_dir:str
-    def __init__(self, name:str="")->None:
+    # A count, for how long a conductor will wait if told that there are no 
+    # jobs in the runner, before polling again. Default is 5 seconds.
+    pause_time: int
+    def __init__(self, name:str="", pause_time:int=5)->None:
         """BaseConductor Constructor. This will check that any class inheriting
         from it implements its validation functions."""
         check_implementation(type(self).execute, BaseConductor)
         check_implementation(type(self).valid_execute_criteria, BaseConductor)
-        check_implementation(type(self).prompt_runner_for_job, BaseConductor)
         if not name:
             name = generate_conductor_id()
         self._is_valid_name(name)
         self.name = name    
+        self._is_valid_pause_time(pause_time)
+        self.pause_time = pause_time
 
     def __new__(cls, *args, **kwargs):
         """A check that this base class is not instantiated itself, only 
@@ -56,21 +63,67 @@ class BaseConductor:
         overridden by child classes."""
         valid_string(name, VALID_CONDUCTOR_NAME_CHARS)
 
-    def prompt_runner_for_job(self):
-        pass
+    def _is_valid_pause_time(self, pause_time:int)->None:
+        """Validation check for 'pause_time' variable from main constructor. Is 
+        automatically called during initialisation. This does not need to be 
+        overridden by child classes."""
+        valid_natural(pause_time, hint="BaseHandler.pause_time")
+
+    def prompt_runner_for_job(self)->Union[Dict[str,Any],Any]:
+        self.to_runner_job.send(1)
+
+        if self.to_runner_job.poll(self.pause_time):
+            return self.to_runner_job.recv()
+        return None
 
     def start(self)->None:
-        """Function to start the conductor as an ongoing process/thread. May be 
-        overidden by any child process. Note that by default this will raise an 
-        execption that is automatically handled within a runner instance."""
-        raise NotImplementedError
+        """Function to start the conductor as an ongoing thread, as defined by 
+        the main_loop function. Together, these will execute any code in a 
+        implemented conductors execute function sequentially, but concurrently 
+        to any other conductors running or other runner operations. This is 
+        intended as a naive mmultiprocessing implementation, and any more in 
+        depth parallelisation of execution must be implemented by a user by 
+        overriding this function, and the stop function."""
+        self._stop_event = Event()        
+        self._handle_thread = Thread(
+            target=self.main_loop, 
+            args=(self._stop_event,),
+            daemon=True,
+            name="conductor_thread"
+        )
+        self._handle_thread.start()
 
     def stop(self)->None:
-        """Function to stop the conductor as an ongoing process/thread. May be 
-        overidden by any child process. Note that by default this will raise an 
-        execption that is automatically handled within a runner instance."""
-        raise NotImplementedError
-    
+        """Function to stop the conductor as an ongoing thread. May be 
+        overidden by any child class. This function should also be overriden if
+        the start function has been."""
+        self._stop_event.set()
+        self._handle_thread.join()
+        
+    def main_loop(self, stop_event)->None:
+        """Function defining an ongoing thread, as started by the start 
+        function and stoped by the stop function. """
+
+        while not stop_event.is_set():
+            reply = self.prompt_runner_for_job()
+
+            # If we have recieved 'None' then we have already timed out so skip 
+            # this loop and start again
+            if reply is None:
+                continue
+
+            try:
+                valid_existing_dir_path(reply)
+            except:
+                # Were not given a job dir, so sleep before trying again
+                sleep(self.pause_time)
+            
+            try:
+                self.execute(reply)
+            except:
+                # TODO some error reporting here
+                pass
+ 
     def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
         """Function to determine given an job defintion, if this conductor can 
         process it or not. Must be implemented by any child process."""