NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ Toy workflow addition

Browse Source
This commit is contained in:
NikolajDanger
2023-06-09 10:57:08 +02:00
parent 52ac5b6576
commit e4b07c385c
10 changed files with 198 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
core/base_handler.py
View File

@ -1,6 +1,6 @@
"""
This file contains the base MEOW handler defintion. This should be inherited
This file contains the base MEOW handler defintion. This should be inherited
from for all handler instances.
Author(s): David Marchant
@ -27,28 +27,28 @@ from meow_base.functionality.meow import create_job_metadata_dict, \
from meow_base.functionality.naming import generate_handler_id
class BaseHandler:
# An identifier for a handler within the runner. Can be manually set in
# An identifier for a handler within the runner. Can be manually set in
# the constructor, or autogenerated if no name provided.
name:str
# A channel for sending messages to the runner event queue. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself,
# A channel for sending messages to the runner event queue. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself,
# unless the handler is running independently of a runner.
to_runner_event: VALID_CHANNELS
# A channel for sending messages to the runner job queue. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself,
# A channel for sending messages to the runner job queue. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself,
# unless the handler is running independently of a runner.
to_runner_job: VALID_CHANNELS
# Directory where queued jobs are initially written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
to_runner_job: VALID_CHANNELS
# Directory where queued jobs are initially written to. Note that this
# will be overridden by a MeowRunner, if a handler instance is passed to
# it, and so does not need to be initialised within the handler itself.
job_queue_dir:str
# A count, for how long a handler will wait if told that there are no
# A count, for how long a handler will wait if told that there are no
# events in the runner, before polling again. Default is 5 seconds.
pause_time: int
def __init__(self, name:str='', pause_time:int=5)->None:
"""BaseHandler Constructor. This will check that any class inheriting
"""BaseHandler Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).valid_handle_criteria, BaseHandler)
check_implementation(type(self).get_created_job_type, BaseHandler)
@ -61,7 +61,7 @@ class BaseHandler:
self.pause_time = pause_time
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseHandler:
msg = get_drt_imp_msg(BaseHandler)
@ -69,14 +69,14 @@ class BaseHandler:
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_HANDLER_NAME_CHARS)
def _is_valid_pause_time(self, pause_time:int)->None:
"""Validation check for 'pause_time' variable from main constructor. Is
automatically called during initialisation. This does not need to be
"""Validation check for 'pause_time' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_natural(pause_time, hint="BaseHandler.pause_time")
@ -91,16 +91,16 @@ class BaseHandler:
self.to_runner_job.send(job_id)
def start(self)->None:
"""Function to start the handler as an ongoing thread, as defined by
the main_loop function. Together, these will execute any code in a
implemented handlers handle function sequentially, but concurrently to
any other handlers running or other runner operations. This is intended
as a naive mmultiprocessing implementation, and any more in depth
parallelisation of execution must be implemented by a user by
"""Function to start the handler as an ongoing thread, as defined by
the main_loop function. Together, these will execute any code in a
implemented handlers handle function sequentially, but concurrently to
any other handlers running or other runner operations. This is intended
as a naive mmultiprocessing implementation, and any more in depth
parallelisation of execution must be implemented by a user by
overriding this function, and the stop function."""
self._stop_event = Event()
self._stop_event = Event()
self._handle_thread = Thread(
target=self.main_loop,
target=self.main_loop,
args=(self._stop_event,),
daemon=True,
name="handler_thread"
@ -108,21 +108,21 @@ class BaseHandler:
self._handle_thread.start()
def stop(self)->None:
"""Function to stop the handler as an ongoing thread. May be overidden
"""Function to stop the handler as an ongoing thread. May be overidden
by any child class. This function should also be overriden if the start
function has been."""
self._stop_event.set()
self._handle_thread.join()
def main_loop(self, stop_event)->None:
"""Function defining an ongoing thread, as started by the start
"""Function defining an ongoing thread, as started by the start
function and stoped by the stop function. """
while not stop_event.is_set():
reply = self.prompt_runner_for_event()
# If we have recieved 'None' then we have already timed out so skip
# If we have recieved 'None' then we have already timed out so skip
# this loop and start again
if reply is None:
continue
@ -138,17 +138,18 @@ class BaseHandler:
self.handle(reply)
except Exception as e:
# TODO some error reporting here
pass
if not isinstance(e, TypeError):
raise e
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
"""Function to determine given an event defintion, if this handler can
process it or not. Must be implemented by any child process."""
pass
def handle(self, event:Dict[str,Any])->None:
"""Function to handle a given event. May be overridden by any child
process. Note that once any handling has occured, the
send_job_to_runner function should be called to inform the runner of
"""Function to handle a given event. May be overridden by any child
process. Note that once any handling has occured, the
send_job_to_runner function should be called to inform the runner of
any resultant jobs."""
rule = event[EVENT_RULE]
@ -158,7 +159,7 @@ class BaseHandler:
yaml_dict[var] = val
for var, val in rule.pattern.outputs.items():
yaml_dict[var] = val
yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH]
# yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH]
# If no parameter sweeps, then one job will suffice
if not rule.pattern.sweep:
@ -172,7 +173,7 @@ class BaseHandler:
self.setup_job(event, yaml_dict)
def setup_job(self, event:Dict[str,Any], params_dict:Dict[str,Any])->None:
"""Function to set up new job dict and send it to the runner to be
"""Function to set up new job dict and send it to the runner to be
executed."""
# Get base job metadata
@ -206,7 +207,7 @@ class BaseHandler:
# TODO make me not tmp variables and update job dict validation
"tmp recipe command": recipe_command,
"tmp script command": script_command
},
},
meta_file
)
@ -216,11 +217,11 @@ class BaseHandler:
def get_created_job_type(self)->str:
pass # Must implemented
def create_job_metadata_dict(self, event:Dict[str,Any],
def create_job_metadata_dict(self, event:Dict[str,Any],
params_dict:Dict[str,Any])->Dict[str,Any]:
return create_job_metadata_dict(
self.get_created_job_type(),
event,
self.get_created_job_type(),
event,
extras={
JOB_PARAMETERS:params_dict
}
@ -253,7 +254,7 @@ class BaseHandler:
"# Check hash of input file to avoid race conditions",
"actual_hash=$(sha256sum $event_path | cut -c -64)",
"echo actual_hash: $actual_hash",
"if [ $given_hash != $actual_hash ]; then",
"if [ \"$given_hash\" != \"$actual_hash\" ]; then",
" echo Job was skipped as triggering file has been modified since scheduling",
" exit 134",
"fi",

22
core/runner.py
View File

@ -156,7 +156,7 @@ class MeowRunner:
f"event for handler {component.name}. {e}",
DEBUG_INFO
)
if valid:
self.event_queue.remove(event)
connection.send(event)
@ -205,7 +205,7 @@ class MeowRunner:
job = threadsafe_read_status(metafile)
except Exception as e:
print_debug(
self._print_target,
self._print_target,
self.debug_level,
"Could not load necessary job definitions "
f"for job at '{job_dir}'. {e}",
@ -216,7 +216,7 @@ class MeowRunner:
valid, _ = component.valid_execute_criteria(job)
except Exception as e:
print_debug(
self._print_target,
self._print_target,
self.debug_level,
"Could not determine validity of "
f"job for conductor {component.name}. {e}",
@ -256,12 +256,12 @@ class MeowRunner:
args=[])
self._mon_han_worker.daemon = True
self._mon_han_worker.start()
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
"Starting MeowRunner event handling...", DEBUG_INFO)
else:
msg = "Repeated calls to start MeowRunner event handling have " \
"no effect."
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
msg, DEBUG_WARNING)
raise RuntimeWarning(msg)
@ -273,12 +273,12 @@ class MeowRunner:
args=[])
self._han_con_worker.daemon = True
self._han_con_worker.start()
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
"Starting MeowRunner job conducting...", DEBUG_INFO)
else:
msg = "Repeated calls to start MeowRunner job conducting have " \
"no effect."
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
msg, DEBUG_WARNING)
raise RuntimeWarning(msg)
@ -302,26 +302,26 @@ class MeowRunner:
# If we've started the monitor/handler interaction thread, then stop it
if self._mon_han_worker is None:
msg = "Cannot stop event handling thread that is not started."
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
msg, DEBUG_WARNING)
raise RuntimeWarning(msg)
else:
self._stop_mon_han_pipe[1].send(1)
self._mon_han_worker.join()
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
"Event handler thread stopped", DEBUG_INFO)
# If we've started the handler/conductor interaction thread, then stop
# it
if self._han_con_worker is None:
msg = "Cannot stop job conducting thread that is not started."
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
msg, DEBUG_WARNING)
raise RuntimeWarning(msg)
else:
self._stop_han_con_pipe[1].send(1)
self._han_con_worker.join()
print_debug(self._print_target, self.debug_level,
print_debug(self._print_target, self.debug_level,
"Job conductor thread stopped", DEBUG_INFO)
def get_monitor_by_name(self, queried_name:str)->BaseMonitor: