NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformatted imports to work better on other machines, plus added benchmarking to project

Browse Source
This commit is contained in:
PatchOfScotland
2023-03-13 11:32:45 +01:00
parent c01df1b190
commit 40ed98000b
48 changed files with 907 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
core/__init__.py Normal file
View File

4
core/base_conductor.py
View File

@ -8,8 +8,8 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from core.correctness.vars import get_drt_imp_msg
from core.correctness.validation import check_implementation
from meow_base.core.correctness.vars import get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation
class BaseConductor:

4
core/base_handler.py
View File

@ -8,8 +8,8 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from core.correctness.validation import check_implementation
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from meow_base.core.correctness.validation import check_implementation
class BaseHandler:

12
core/base_monitor.py
View File

@ -9,12 +9,12 @@ Author(s): David Marchant
from copy import deepcopy
from typing import Union, Dict
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from core.correctness.validation import check_implementation
from functionality.meow import create_rules
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from meow_base.core.correctness.validation import check_implementation
from meow_base.functionality.meow import create_rules
class BaseMonitor:

4
core/base_pattern.py
View File

@ -10,9 +10,9 @@ import itertools
from typing import Any, Union, Tuple, Dict, List
from core.correctness.vars import get_drt_imp_msg, \
from meow_base.core.correctness.vars import get_drt_imp_msg, \
VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
from core.correctness.validation import valid_string, check_type, \
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation, valid_dict

4
core/base_recipe.py
View File

@ -8,8 +8,8 @@ Author(s): David Marchant
from typing import Any, Dict
from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
from core.correctness.validation import valid_string, check_implementation
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
from meow_base.core.correctness.validation import valid_string, check_implementation
class BaseRecipe:

8
core/base_rule.py
View File

@ -10,11 +10,11 @@ from sys import modules
from typing import Any
if "BasePattern" not in modules:
from core.base_pattern import BasePattern
from meow_base.core.base_pattern import BasePattern
if "BaseRecipe" not in modules:
from core.base_recipe import BaseRecipe
from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
from core.correctness.validation import valid_string, check_type, \
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation

6
core/correctness/meow.py
View File

@ -2,9 +2,9 @@
from datetime import datetime
from typing import Any, Dict, Type
from core.base_rule import BaseRule
from core.correctness.validation import check_type
from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type
from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \
JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH

2
core/correctness/validation.py
View File

@ -11,7 +11,7 @@ from os.path import sep, exists, isfile, isdir, dirname
from typing import Any, _SpecialForm, Union, Type, Dict, List, \
get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
from meow_base.core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
or_none:bool=False, hint:str="")->None:

49
core/runner.py
View File

@ -14,16 +14,16 @@ from multiprocessing import Pipe
from random import randrange
from typing import Any, Union, Dict, List
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR, \
EVENT_PATH
from core.correctness.validation import check_type, valid_list, valid_dir_path
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_yaml
from functionality.process_io import wait
from meow_base.core.correctness.validation import check_type, valid_list, valid_dir_path
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_yaml
from meow_base.functionality.process_io import wait
class MeowRunner:
@ -112,6 +112,7 @@ class MeowRunner:
if self._stop_mon_han_pipe[0] in ready:
return
else:
handled = False
for from_monitor in self.from_monitors:
if from_monitor in ready:
# Read event from the monitor channel
@ -136,13 +137,25 @@ class MeowRunner:
# If we've only one handler, use that
if len(valid_handlers) == 1:
handler = valid_handlers[0]
handled = True
self.handle_event(handler, event)
break
# If multiple handlers then randomly pick one
else:
elif len(valid_handlers) > 1:
handler = valid_handlers[
randrange(len(valid_handlers))
]
handled = True
self.handle_event(handler, event)
break
if not handled:
print_debug(
self._print_target,
self.debug_level,
"Could not determine handler for event.",
DEBUG_INFO
)
def run_handler_conductor_interaction(self)->None:
"""Function to be run in its own thread, to handle any inbound messages
@ -156,6 +169,7 @@ class MeowRunner:
if self._stop_han_con_pipe[0] in ready:
return
else:
executed = False
for from_handler in self.from_handlers:
if from_handler in ready:
# Read job directory from the handler channel
@ -188,14 +202,27 @@ class MeowRunner:
# If we've only one conductor, use that
if len(valid_conductors) == 1:
conductor = valid_conductors[0]
executed = True
self.execute_job(conductor, job_dir)
break
# If multiple handlers then randomly pick one
else:
elif len(valid_conductors) > 1:
conductor = valid_conductors[
randrange(len(valid_conductors))
]
executed = True
self.execute_job(conductor, job_dir)
break
# TODO determine something more useful to do here
if not executed:
print_debug(
self._print_target,
self.debug_level,
f"No conductor could be found for job {job_dir}",
DEBUG_INFO
)
def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None:
"""Function for a given handler to handle a given event, without
crashing the runner in the event of a problem."""