updated runner structure so that handlers and conductors actually pull from queues in the runner. changes to logic in both are extensive, but most individual functinos are unaffected. I've also moved several functions that were part of individual monitor, handler and conductors to the base classes.

This commit is contained in:
PatchOfScotland
2023-04-20 17:08:06 +02:00
parent b87fd43cfd
commit f306d8b6f2
16 changed files with 1589 additions and 964 deletions

View File

@ -61,13 +61,13 @@ class PythonHandler(BaseHandler):
# Where print messages are sent
_print_target:Any
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
print:Any=sys.stdout, logging:int=0)->None:
print:Any=sys.stdout, logging:int=0, pause_time:int=5)->None:
"""PythonHandler Constructor. This creates jobs to be executed as
python functions. This does not run as a continuous thread to
handle execution, but is invoked according to a factory pattern using
the handle function. Note that if this handler is given to a MeowRunner
object, the job_queue_dir will be overwridden by its"""
super().__init__(name=name)
super().__init__(name=name, pause_time=pause_time)
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir
self._print_target, self.debug_level = setup_debugging(print, logging)
@ -173,7 +173,7 @@ class PythonHandler(BaseHandler):
)
# Send job directory, as actual definitons will be read from within it
self.send_to_runner(job_dir)
self.send_job_to_runner(job_dir)
# Papermill job execution code, to be run within the conductor