NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added python handler, and reworked handler and conductor event/job discovery to be more modular

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-01 17:43:16 +01:00
parent 5acb8c230e
commit 636d70f4e8
17 changed files with 537 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
core/correctness/validation.py
View File

@ -12,12 +12,15 @@ from typing import Any, _SpecialForm, Union, Tuple, get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg, \
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE
# Required keys in event dict
EVENT_KEYS = {
EVENT_TYPE: str,
EVENT_PATH: str
EVENT_PATH: str,
# TODO sort this
# Should be a Rule but can't import here due to circular dependencies
EVENT_RULE: Any
}
# Required keys in job dict

8
core/correctness/vars.py
View File

@ -41,9 +41,11 @@ SHA256 = "sha256"
# meow events
EVENT_TYPE = "event_type"
EVENT_PATH = "event_path"
WATCHDOG_TYPE = "watchdog"
EVENT_RULE = "rule"
# watchdog events
EVENT_TYPE_WATCHDOG = "watchdog"
WATCHDOG_BASE = "monitor_base"
WATCHDOG_RULE = "rule_name"
WATCHDOG_HASH = "file_hash"
# inotify events
@ -77,7 +79,7 @@ DIR_EVENTS = [
# meow jobs
JOB_TYPE = "job_type"
PYTHON_TYPE = "python"
JOB_TYPE_PYTHON = "python"
PYTHON_FUNC = "func"
PYTHON_EXECUTION_BASE = "exection_base"
PYTHON_OUTPUT_DIR = "output_dir"

19
core/functionality.py
View File

@ -19,7 +19,7 @@ from core.correctness.validation import check_type, valid_existing_file_path, \
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
VALID_CHANNELS, HASH_BUFFER_SIZE, SHA256, DEBUG_WARNING, DEBUG_INFO, \
EVENT_TYPE, EVENT_PATH, JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, WATCHDOG_RULE, JOB_STATUS, STATUS_QUEUED, \
JOB_RECIPE, JOB_RULE, EVENT_RULE, JOB_STATUS, STATUS_QUEUED, \
JOB_CREATE_TIME, JOB_REQUIREMENTS
# mig trigger keyword replacements
@ -283,9 +283,14 @@ def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
return new_dict
def create_event(event_type:str, path:str, source:dict[Any,Any]={}
def create_event(event_type:str, path:str, rule:Any, source:dict[Any,Any]={}
)->dict[Any,Any]:
return {**source, EVENT_PATH: path, EVENT_TYPE: event_type}
return {
**source,
EVENT_PATH: path,
EVENT_TYPE: event_type,
EVENT_RULE: rule
}
def create_job(job_type:str, event:dict[str,Any], source:dict[Any,Any]={}
)->dict[Any,Any]:
@ -294,12 +299,12 @@ def create_job(job_type:str, event:dict[str,Any], source:dict[Any,Any]={}
JOB_ID: generate_id(prefix="job_"),
JOB_EVENT: event,
JOB_TYPE: job_type,
JOB_PATTERN: event[WATCHDOG_RULE].pattern.name,
JOB_RECIPE: event[WATCHDOG_RULE].recipe.name,
JOB_RULE: event[WATCHDOG_RULE].name,
JOB_PATTERN: event[EVENT_RULE].pattern.name,
JOB_RECIPE: event[EVENT_RULE].recipe.name,
JOB_RULE: event[EVENT_RULE].name,
JOB_STATUS: STATUS_QUEUED,
JOB_CREATE_TIME: datetime.now(),
JOB_REQUIREMENTS: event[WATCHDOG_RULE].recipe.requirements
JOB_REQUIREMENTS: event[EVENT_RULE].recipe.requirements
}
return {**source, **job_dict}

17
core/meow.py
View File

@ -313,7 +313,7 @@ class BaseHandler:
"""BaseHandler Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).handle, BaseHandler)
check_implementation(type(self).valid_event_types, BaseHandler)
check_implementation(type(self).valid_handle_criteria, BaseHandler)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
@ -323,9 +323,10 @@ class BaseHandler:
raise TypeError(msg)
return object.__new__(cls)
def valid_event_types(self)->list[str]:
"""Function to provide a list of the types of events this handler can
process. Must be implemented by any child process."""
# TODO also implement something like me from conductor
def valid_handle_criteria(self, event:dict[str,Any])->bool:
"""Function to determine given an event defintion, if this handler can
process it or not. Must be implemented by any child process."""
pass
def handle(self, event:dict[str,Any])->None:
@ -339,7 +340,7 @@ class BaseConductor:
"""BaseConductor Constructor. This will check that any class inheriting
from it implements its validation functions."""
check_implementation(type(self).execute, BaseConductor)
check_implementation(type(self).valid_job_types, BaseConductor)
check_implementation(type(self).valid_execute_criteria, BaseConductor)
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
@ -349,9 +350,9 @@ class BaseConductor:
raise TypeError(msg)
return object.__new__(cls)
def valid_job_types(self)->list[str]:
"""Function to provide a list of the types of jobs this conductor can
process. Must be implemented by any child process."""
def valid_execute_criteria(self, job:dict[str,Any])->bool:
"""Function to determine given an job defintion, if this conductor can
process it or not. Must be implemented by any child process."""
pass
def execute(self, job:dict[str,Any])->None:

129
core/runner.py
View File

@ -27,9 +27,9 @@ class MeowRunner:
# A collection of all monitors in the runner
monitors:list[BaseMonitor]
# A collection of all handlers in the runner
handlers:dict[str:BaseHandler]
handlers:list[BaseHandler]
# A collection of all conductors in the runner
conductors:dict[str:BaseConductor]
conductors:list[BaseConductor]
# A collection of all channels from each monitor
from_monitors: list[VALID_CHANNELS]
# A collection of all channels from each handler
@ -46,48 +46,19 @@ class MeowRunner:
# If conductors isn't a list, make it one
if not type(conductors) == list:
conductors = [conductors]
self.conductors = {}
# Create a dictionary of conductors, keyed by job type, and valued by a
# list of conductors for that job type
for conductor in conductors:
conductor_jobs = conductor.valid_job_types()
if not conductor_jobs:
raise ValueError(
"Cannot start runner with conductor that does not "
f"implement '{BaseConductor.valid_job_types.__name__}"
f"({signature(BaseConductor.valid_job_types)})' and "
"return a list of at least one conductable job.")
for job in conductor_jobs:
if job in self.conductors.keys():
self.conductors[job].append(conductor)
else:
self.conductors[job] = [conductor]
self.conductors = conductors
self._is_valid_handlers(handlers)
# If handlers isn't a list, make it one
if not type(handlers) == list:
handlers = [handlers]
self.handlers = {}
self.from_handlers = []
# Create a dictionary of handlers, keyed by event type, and valued by a
# list of handlers for that event type
for handler in handlers:
handler_events = handler.valid_event_types()
if not handler_events:
raise ValueError(
"Cannot start runner with handler that does not "
f"implement '{BaseHandler.valid_event_types.__name__}"
f"({signature(BaseHandler.valid_event_types)})' and "
"return a list of at least one handlable event.")
for event in handler_events:
if event in self.handlers.keys():
self.handlers[event].append(handler)
else:
self.handlers[event] = [handler]
for handler in handlers:
# Create a channel from the handler back to this runner
handler_to_runner_reader, handler_to_runner_writer = Pipe()
handler.to_runner = handler_to_runner_writer
self.from_handlers.append(handler_to_runner_reader)
self.handlers = handlers
self._is_valid_monitors(monitors)
# If monitors isn't a list, make it one
@ -129,21 +100,30 @@ class MeowRunner:
# Read event from the monitor channel
message = from_monitor.recv()
event = message
# Abort if we don't have a relevent handler.
if not self.handlers[event[EVENT_TYPE]]:
print_debug(self._print_target, self.debug_level,
"Could not process event as no relevent "
f"handler for '{event[EVENT_TYPE]}'",
DEBUG_INFO)
continue
valid_handlers = []
for handler in self.handlers:
try:
valid = handler.valid_handle_criteria(event)
if valid:
valid_handlers.append(handler)
except Exception as e:
print_debug(
self._print_target,
self.debug_level,
"Could not determine validity of event "
f"for handler. {e}",
DEBUG_INFO
)
# If we've only one handler, use that
if len(self.handlers[event[EVENT_TYPE]]) == 1:
handler = self.handlers[event[EVENT_TYPE]][0]
if len(valid_handlers) == 1:
handler = valid_handlers[0]
self.handle_event(handler, event)
# If multiple handlers then randomly pick one
else:
handler = self.handlers[event[EVENT_TYPE]][
randrange(len(self.handlers[event[EVENT_TYPE]]))
handler = valid_handlers[
randrange(len(valid_handlers))
]
self.handle_event(handler, event)
@ -172,20 +152,29 @@ class MeowRunner:
f"job at '{job_dir}'. {e}", DEBUG_INFO)
continue
# Abort if we don't have a relevent conductor.
if not self.conductors[job[JOB_TYPE]]:
print_debug(self._print_target, self.debug_level,
"Could not process job as no relevent "
f"conductor for '{job[JOB_TYPE]}'", DEBUG_INFO)
continue
valid_conductors = []
for conductor in self.conductors:
try:
valid = conductor.valid_execute_criteria(job)
if valid:
valid_conductors.append(conductor)
except Exception as e:
print_debug(
self._print_target,
self.debug_level,
"Could not determine validity of job "
f"for conductor. {e}",
DEBUG_INFO
)
# If we've only one conductor, use that
if len(self.conductors[job[JOB_TYPE]]) == 1:
conductor = self.conductors[job[JOB_TYPE]][0]
if len(valid_conductors) == 1:
conductor = valid_conductors[0]
self.execute_job(conductor, job)
# If multiple conductors then randomly pick one
# If multiple handlers then randomly pick one
else:
conductor = self.conductors[job[JOB_TYPE]][
randrange(len(self.conductors[job[JOB_TYPE]]))
conductor = valid_conductors[
randrange(len(valid_conductors))
]
self.execute_job(conductor, job)
@ -227,15 +216,13 @@ class MeowRunner:
monitor.start()
startable = []
# Start all handlers, if they need it
for handler_list in self.handlers.values():
for handler in handler_list:
if hasattr(handler, "start") and handler not in startable:
startable.append()
for handler in self.handlers:
if hasattr(handler, "start") and handler not in startable:
startable.append()
# Start all conductors, if they need it
for conductor_list in self.conductors.values():
for conductor in conductor_list:
if hasattr(conductor, "start") and conductor not in startable:
startable.append()
for conductor in self.conductors:
if hasattr(conductor, "start") and conductor not in startable:
startable.append()
for starting in startable:
starting.start()
@ -283,15 +270,13 @@ class MeowRunner:
stopable = []
# Stop all handlers, if they need it
for handler_list in self.handlers.values():
for handler in handler_list:
if hasattr(handler, "stop") and handler not in stopable:
stopable.append()
for handler in self.handlers:
if hasattr(handler, "stop") and handler not in stopable:
stopable.append()
# Stop all conductors, if they need it
for conductor_list in self.conductors.values():
for conductor in conductor_list:
if hasattr(conductor, "stop") and conductor not in stopable:
stopable.append()
for conductor in self.conductors:
if hasattr(conductor, "stop") and conductor not in stopable:
stopable.append()
for stopping in stopable:
stopping.stop()