""" This file contains the base MEOW conductor defintion. This should be inherited from for all conductor instances. Author(s): David Marchant """ from threading import Event, Thread from time import sleep from typing import Any, Tuple, Dict, Union from meow_base.core.vars import VALID_CONDUCTOR_NAME_CHARS, VALID_CHANNELS, \ get_drt_imp_msg from meow_base.functionality.validation import check_implementation, \ valid_string, valid_existing_dir_path, valid_natural from meow_base.functionality.naming import generate_conductor_id class BaseConductor: # An identifier for a conductor within the runner. Can be manually set in # the constructor, or autogenerated if no name provided. name:str # A channel for sending messages to the runner job queue. Note that this # will be overridden by a MeowRunner, if a conductor instance is passed to # it, and so does not need to be initialised within the conductor itself, # unless the conductor is running independently of a runner. to_runner_job: VALID_CHANNELS # Directory where queued jobs are initially written to. Note that this # will be overridden by a MeowRunner, if a handler instance is passed to # it, and so does not need to be initialised within the handler itself. job_queue_dir:str # Directory where completed jobs are finally written to. Note that this # will be overridden by a MeowRunner, if a handler instance is passed to # it, and so does not need to be initialised within the handler itself. job_output_dir:str # A count, for how long a conductor will wait if told that there are no # jobs in the runner, before polling again. Default is 5 seconds. pause_time: int def __init__(self, name:str="", pause_time:int=5)->None: """BaseConductor Constructor. This will check that any class inheriting from it implements its validation functions.""" check_implementation(type(self).execute, BaseConductor) check_implementation(type(self).valid_execute_criteria, BaseConductor) if not name: name = generate_conductor_id() self._is_valid_name(name) self.name = name self._is_valid_pause_time(pause_time) self.pause_time = pause_time def __new__(cls, *args, **kwargs): """A check that this base class is not instantiated itself, only inherited from""" if cls is BaseConductor: msg = get_drt_imp_msg(BaseConductor) raise TypeError(msg) return object.__new__(cls) def _is_valid_name(self, name:str)->None: """Validation check for 'name' variable from main constructor. Is automatically called during initialisation. This does not need to be overridden by child classes.""" valid_string(name, VALID_CONDUCTOR_NAME_CHARS) def _is_valid_pause_time(self, pause_time:int)->None: """Validation check for 'pause_time' variable from main constructor. Is automatically called during initialisation. This does not need to be overridden by child classes.""" valid_natural(pause_time, hint="BaseHandler.pause_time") def prompt_runner_for_job(self)->Union[Dict[str,Any],Any]: self.to_runner_job.send(1) if self.to_runner_job.poll(self.pause_time): return self.to_runner_job.recv() return None def start(self)->None: """Function to start the conductor as an ongoing thread, as defined by the main_loop function. Together, these will execute any code in a implemented conductors execute function sequentially, but concurrently to any other conductors running or other runner operations. This is intended as a naive mmultiprocessing implementation, and any more in depth parallelisation of execution must be implemented by a user by overriding this function, and the stop function.""" self._stop_event = Event() self._handle_thread = Thread( target=self.main_loop, args=(self._stop_event,), daemon=True, name="conductor_thread" ) self._handle_thread.start() def stop(self)->None: """Function to stop the conductor as an ongoing thread. May be overidden by any child class. This function should also be overriden if the start function has been.""" self._stop_event.set() self._handle_thread.join() def main_loop(self, stop_event)->None: """Function defining an ongoing thread, as started by the start function and stoped by the stop function. """ while not stop_event.is_set(): reply = self.prompt_runner_for_job() # If we have recieved 'None' then we have already timed out so skip # this loop and start again if reply is None: continue try: valid_existing_dir_path(reply) except: # Were not given a job dir, so sleep before trying again sleep(self.pause_time) try: self.execute(reply) except: # TODO some error reporting here pass def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: """Function to determine given an job defintion, if this conductor can process it or not. Must be implemented by any child process.""" pass def execute(self, job_dir:str)->None: """Function to execute a given job directory. Must be implemented by any child process.""" pass