NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rewored rules to only invoke base rule, and added bash jobs

Browse Source
This commit is contained in:
PatchOfScotland
2023-03-30 11:33:15 +02:00
parent 747f2c316c
commit 311c98f7f2
20 changed files with 1046 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
conductors/local_bash_conductor.py Normal file
View File

@ -0,0 +1,129 @@
"""
This file contains definitions for the LocalBashConductor, in order to
execute Bash jobs on the local resource.
Author(s): David Marchant
"""
import os
import shutil
import subprocess
from datetime import datetime
from typing import Any, Dict, Tuple
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import DEFAULT_JOB_QUEUE_DIR, \
DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, META_FILE, JOB_STATUS, \
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
JOB_ERROR, JOB_TYPE, DEFAULT_JOB_QUEUE_DIR, STATUS_RUNNING, \
JOB_START_TIME, DEFAULT_JOB_OUTPUT_DIR, get_job_file
from meow_base.core.correctness.validation import valid_dir_path
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
write_yaml
class LocalBashConductor(BaseConductor):
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR,
job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="")->None:
"""LocalBashConductor Constructor. This should be used to execute
Bash jobs, and will then pass any internal job runner files to the
output directory. Note that if this handler is given to a MeowRunner
object, the job_queue_dir and job_output_dir will be overwridden."""
super().__init__(name=name)
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir
self._is_valid_job_output_dir(job_output_dir)
self.job_output_dir = job_output_dir
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
process it or not. This conductor will accept any Bash job type"""
try:
valid_job(job)
msg = ""
if job[JOB_TYPE] not in [JOB_TYPE_BASH]:
msg = f"Job type was not {JOB_TYPE_BASH}."
if msg:
return False, msg
else:
return True, ""
except Exception as e:
return False, str(e)
def execute(self, job_dir:str)->None:
"""Function to actually execute a Bash job. This will read job
defintions from its meta file, update the meta file and attempt to
execute. Some unspecific feedback will be given on execution failure,
but depending on what it is it may be up to the job itself to provide
more detailed feedback."""
valid_dir_path(job_dir, must_exist=True)
# Test our job parameters. Even if its gibberish, we still move to
# output
abort = False
try:
meta_file = os.path.join(job_dir, META_FILE)
job = read_yaml(meta_file)
valid_job(job)
# update the status file with running status
job[JOB_STATUS] = STATUS_RUNNING
job[JOB_START_TIME] = datetime.now()
write_yaml(job, meta_file)
except Exception as e:
# If something has gone wrong at this stage then its bad, so we
# need to make our own error file
error_file = os.path.join(job_dir, BACKUP_JOB_ERROR_FILE)
write_file(f"Recieved incorrectly setup job.\n\n{e}", error_file)
abort = True
# execute the job
if not abort:
try:
result = subprocess.call(get_job_file(JOB_TYPE_BASH), cwd=".")
# get up to date job data
job = read_yaml(meta_file)
# Update the status file with the finalised status
job[JOB_STATUS] = STATUS_DONE
job[JOB_END_TIME] = datetime.now()
write_yaml(job, meta_file)
except Exception as e:
# get up to date job data
job = read_yaml(meta_file)
# Update the status file with the error status. Don't overwrite
# any more specific error messages already created
if JOB_STATUS not in job:
job[JOB_STATUS] = STATUS_FAILED
if JOB_END_TIME not in job:
job[JOB_END_TIME] = datetime.now()
if JOB_ERROR not in job:
job[JOB_ERROR] = f"Job execution failed. {e}"
write_yaml(job, meta_file)
# Move the contents of the execution directory to the final output
# directory.
job_output_dir = \
os.path.join(self.job_output_dir, os.path.basename(job_dir))
shutil.move(job_dir, job_output_dir)
def _is_valid_job_queue_dir(self, job_queue_dir)->None:
"""Validation check for 'job_queue_dir' variable from main
constructor."""
valid_dir_path(job_queue_dir, must_exist=False)
if not os.path.exists(job_queue_dir):
make_dir(job_queue_dir)
def _is_valid_job_output_dir(self, job_output_dir)->None:
"""Validation check for 'job_output_dir' variable from main
constructor."""
valid_dir_path(job_output_dir, must_exist=False)
if not os.path.exists(job_output_dir):
make_dir(job_output_dir)

6
core/base_monitor.py
View File

@ -11,7 +11,7 @@ from typing import Union, Dict
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.rule import Rule
from meow_base.core.correctness.vars import VALID_CHANNELS, \
VALID_MONITOR_NAME_CHARS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
@ -29,7 +29,7 @@ class BaseMonitor:
# A collection of recipes
_recipes: Dict[str, BaseRecipe]
# A collection of rules derived from _patterns and _recipes
_rules: Dict[str, BaseRule]
_rules: Dict[str, Rule]
# A channel for sending messages to the runner. Note that this is not
# initialised within the constructor, but within the runner when passed the
# monitor is passed to it.
@ -138,7 +138,7 @@ class BaseMonitor:
Must be implemented by any child process."""
pass
def get_rules(self)->Dict[str,BaseRule]:
def get_rules(self)->Dict[str,Rule]:
"""Function to get a dictionary of all current rule definitions.
Must be implemented by any child process."""
pass

85
core/base_rule.py
View File

@ -1,85 +0,0 @@
"""
This file contains the base MEOW rule defintion. This should be inherited from
for all rule instances.
Author(s): David Marchant
"""
from sys import modules
from typing import Any
if "BasePattern" not in modules:
from meow_base.core.base_pattern import BasePattern
if "BaseRecipe" not in modules:
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation
class BaseRule:
# A unique identifier for the rule
name:str
# A pattern to be used in rule triggering
pattern:BasePattern
# A recipe to be used in rule execution
recipe:BaseRecipe
# The string name of the pattern class that can be used to create this rule
pattern_type:str=""
# The string name of the recipe class that can be used to create this rule
recipe_type:str=""
def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
"""BaseRule Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
check_implementation(type(self)._is_valid_pattern, BaseRule)
check_implementation(type(self)._is_valid_recipe, BaseRule)
self.__check_types_set()
self._is_valid_name(name)
self.name = name
self._is_valid_pattern(pattern)
self.pattern = pattern
self._is_valid_recipe(recipe)
self.recipe = recipe
check_type(pattern, BasePattern, hint="BaseRule.pattern")
check_type(recipe, BaseRecipe, hint="BaseRule.recipe")
if pattern.recipe != recipe.name:
raise ValueError(f"Cannot create Rule {name}. Pattern "
f"{pattern.name} does not identify Recipe {recipe.name}. It "
f"uses {pattern.recipe}")
def __new__(cls, *args, **kwargs):
"""A check that this base class is not instantiated itself, only
inherited from"""
if cls is BaseRule:
msg = get_drt_imp_msg(BaseRule)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation. This does not need to be
overridden by child classes."""
valid_string(name, VALID_RULE_NAME_CHARS)
def _is_valid_pattern(self, pattern:Any)->None:
"""Validation check for 'pattern' variable from main constructor. Must
be implemented by any child class."""
pass
def _is_valid_recipe(self, recipe:Any)->None:
"""Validation check for 'recipe' variable from main constructor. Must
be implemented by any child class."""
pass
def __check_types_set(self)->None:
"""Validation check that the self.pattern_type and self.recipe_type
attributes have been set in a child class."""
if self.pattern_type == "":
raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
"does not set a pattern_type.")
if self.recipe_type == "":
raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
"does not set a recipe_type.")

4
core/correctness/meow.py
View File

@ -2,7 +2,7 @@
from datetime import datetime
from typing import Any, Dict, Type
from meow_base.core.base_rule import BaseRule
from meow_base.core.rule import Rule
from meow_base.core.correctness.validation import check_type
from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \
JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
@ -13,7 +13,7 @@ EVENT_KEYS = {
EVENT_TYPE: str,
EVENT_PATH: str,
# Should be a Rule but can't import here due to circular dependencies
EVENT_RULE: BaseRule
EVENT_RULE: Rule
}
WATCHDOG_EVENT_KEYS = {

10
core/correctness/vars.py
View File

@ -86,6 +86,7 @@ DEFAULT_JOB_OUTPUT_DIR = "job_output"
# meow jobs
JOB_TYPE = "job_type"
JOB_TYPE_BASH = "bash"
JOB_TYPE_PYTHON = "python"
JOB_TYPE_PAPERMILL = "papermill"
PYTHON_FUNC = "func"
@ -94,12 +95,17 @@ JOB_TYPES = {
JOB_TYPE_PAPERMILL: [
"base.ipynb",
"job.ipynb",
"result.ipynb",
"result.ipynb"
],
JOB_TYPE_PYTHON: [
"base.py",
"job.py",
"result.py",
"result.py"
],
JOB_TYPE_BASH: [
"base.sh",
"job.sh",
"result.sh"
]
}

49
core/rule.py Normal file
View File

@ -0,0 +1,49 @@
"""
This file contains the MEOW rule defintion.
Author(s): David Marchant
"""
from sys import modules
from typing import Any
if "BasePattern" not in modules:
from meow_base.core.base_pattern import BasePattern
if "BaseRecipe" not in modules:
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation
from meow_base.functionality.naming import generate_rule_id
class Rule:
# A unique identifier for the rule
name:str
# A pattern to be used in rule triggering
pattern:BasePattern
# A recipe to be used in rule execution
recipe:BaseRecipe
def __init__(self, pattern:BasePattern, recipe:BaseRecipe, name:str=""):
"""Rule Constructor. This will check that any class inheriting
from it implements its validation functions. It will then call these on
the input parameters."""
if not name:
name = generate_rule_id()
self._is_valid_name(name)
self.name = name
check_type(pattern, BasePattern, hint="Rule.pattern")
self.pattern = pattern
check_type(recipe, BaseRecipe, hint="Rule.recipe")
self.recipe = recipe
if pattern.recipe != recipe.name:
raise ValueError(f"Cannot create Rule {name}. Pattern "
f"{pattern.name} does not identify Recipe {recipe.name}. It "
f"uses {pattern.recipe}")
def _is_valid_name(self, name:str)->None:
"""Validation check for 'name' variable from main constructor. Is
automatically called during initialisation."""
valid_string(name, VALID_RULE_NAME_CHARS)

40
functionality/meow.py
View File

@ -10,7 +10,7 @@ from typing import Any, Dict, Union, List
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.rule import Rule
from meow_base.core.correctness.validation import check_type, valid_dict, \
valid_list
from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \
@ -18,7 +18,7 @@ from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \
JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \
SWEEP_START, SWEEP_STOP
from meow_base.functionality.naming import generate_job_id, generate_rule_id
from meow_base.functionality.naming import generate_job_id
# mig trigger keyword replacements
KEYWORD_PATH = "{PATH}"
@ -147,8 +147,7 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
return {**extras, **job_dict}
def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]],
new_rules:List[BaseRule]=[])->Dict[str,BaseRule]:
recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]])->Dict[str,Rule]:
"""Function to create any valid rules from a given collection of patterns
and recipes. All inbuilt rule types are considered, with additional
definitions provided through the 'new_rules' variable. Note that any
@ -157,7 +156,6 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
# Validation of inputs
check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns")
check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes")
valid_list(new_rules, BaseRule, min_length=0)
# Convert a pattern list to a dictionary
if isinstance(patterns, list):
@ -198,34 +196,14 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]],
pass
return generated_rules
def create_rule(pattern:BasePattern, recipe:BaseRecipe,
new_rules:List[BaseRule]=[])->BaseRule:
def create_rule(pattern:BasePattern, recipe:BaseRecipe)->Rule:
"""Function to create a valid rule from a given pattern and recipe. All
inbuilt rule types are considered, with additional definitions provided
through the 'new_rules' variable."""
check_type(pattern, BasePattern, hint="create_rule.pattern")
check_type(recipe, BaseRecipe, hint="create_rule.recipe")
valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules")
# TODO fix me
# Imported here to avoid circular imports at top of file
import meow_base.rules
all_rules = {
(r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()
}
# Add in new rules
for rule in new_rules:
all_rules[(rule.pattern_type, rule.recipe_type)] = rule
# Find appropriate rule type from pattern and recipe types
key = (type(pattern).__name__, type(recipe).__name__)
if (key) in all_rules:
return all_rules[key](
generate_rule_id(),
pattern,
recipe
)
# Raise error if not valid rule type can be found
raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe "
f"'{recipe}' could be found.")
return Rule(
pattern,
recipe
)

33
functionality/parameterisation.py
View File

@ -119,3 +119,36 @@ def parameterize_python_script(script:List[str], parameters:Dict[str,Any],
check_script(output_script)
return output_script
def parameterize_bash_script(script:List[str], parameters:Dict[str,Any],
expand_env_values:bool=False)->Dict[str,Any]:
check_script(script)
check_type(parameters, Dict
,hint="parameterize_bash_script.parameters")
output_script = deepcopy(script)
for i, line in enumerate(output_script):
if "=" in line:
d_line = list(map(lambda x: x.replace(" ", ""),
line.split("=")))
# Matching parameter name
if len(d_line) == 2 and d_line[0] in parameters:
value = parameters[d_line[0]]
# Whether to expand value from os env
if (
expand_env_values
and isinstance(value, str)
and value.startswith("ENV_")
):
env_var = value.replace("ENV_", "")
value = getenv(
env_var,
"MISSING ENVIRONMENT VARIABLE: {}".format(env_var)
)
output_script[i] = f"{d_line[0]}={repr(value)}"
# Validate that the parameterized notebook is still valid
check_script(output_script)
return output_script

6
patterns/file_event_pattern.py
View File

@ -21,7 +21,7 @@ from watchdog.events import PatternMatchingEventHandler
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_rule import BaseRule
from meow_base.core.rule import Rule
from meow_base.core.correctness.validation import check_type, valid_string, \
valid_dict, valid_list, valid_dir_path
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
@ -358,7 +358,7 @@ class WatchdogMonitor(BaseMonitor):
self._recipes_lock.release()
return to_return
def get_rules(self)->Dict[str,BaseRule]:
def get_rules(self)->Dict[str,Rule]:
"""Function to get a dict of the currently defined rules of the
monitor. Note that the result is deep-copied, and so can be manipulated
without directly manipulating the internals of the monitor."""
@ -486,7 +486,7 @@ class WatchdogMonitor(BaseMonitor):
for rule in self._rules.values():
self._apply_retroactive_rule(rule)
def _apply_retroactive_rule(self, rule:BaseRule)->None:
def _apply_retroactive_rule(self, rule:Rule)->None:
"""Function to determine if a rule should be applied to the existing
file structure, were the file structure created/modified now."""
self._rules_lock.acquire()

212
recipes/bash_recipe.py Normal file
View File

@ -0,0 +1,212 @@
import os
import stat
import sys
from typing import Any, Dict, List, Tuple
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_type, valid_dict, \
valid_string, valid_dir_path
from meow_base.core.correctness.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
VALID_VARIABLE_NAME_CHARS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, JOB_ID, \
EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_PARAMETERS, JOB_HASH, \
WATCHDOG_HASH, WATCHDOG_BASE, META_FILE, PARAMS_FILE, STATUS_QUEUED, \
JOB_STATUS, \
get_base_file, get_job_file
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import valid_path, make_dir, write_yaml, \
write_file, lines_to_string
from meow_base.functionality.parameterisation import parameterize_bash_script
from meow_base.functionality.meow import create_job, replace_keywords
class BashRecipe(BaseRecipe):
# A path to the bash script used to create this recipe
def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={},
requirements:Dict[str,Any]={}, source:str=""):
"""BashRecipe Constructor. This is used to execute bash scripts,
enabling anything not natively supported by MEOW."""
super().__init__(name, recipe, parameters, requirements)
self._is_valid_source(source)
self.source = source
def _is_valid_source(self, source:str)->None:
"""Validation check for 'source' variable from main constructor."""
if source:
valid_path(source, extension=".sh", min_length=0)
def _is_valid_recipe(self, recipe:List[str])->None:
"""Validation check for 'recipe' variable from main constructor.
Called within parent BaseRecipe constructor."""
check_type(recipe, List, hint="BashRecipe.recipe")
for line in recipe:
check_type(line, str, hint="BashRecipe.recipe[line]")
def _is_valid_parameters(self, parameters:Dict[str,Any])->None:
"""Validation check for 'parameters' variable from main constructor.
Called within parent BaseRecipe constructor."""
valid_dict(parameters, str, Any, strict=False, min_length=0)
for k in parameters.keys():
valid_string(k, VALID_VARIABLE_NAME_CHARS)
def _is_valid_requirements(self, requirements:Dict[str,Any])->None:
"""Validation check for 'requirements' variable from main constructor.
Called within parent BaseRecipe constructor."""
valid_dict(requirements, str, Any, strict=False, min_length=0)
for k in requirements.keys():
valid_string(k, VALID_VARIABLE_NAME_CHARS)
class BashHandler(BaseHandler):
# Config option, above which debug messages are ignored
debug_level:int
# Where print messages are sent
_print_target:Any
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="",
print:Any=sys.stdout, logging:int=0)->None:
"""BashHandler Constructor. This creates jobs to be executed as
bash scripts. This does not run as a continuous thread to
handle execution, but is invoked according to a factory pattern using
the handle function. Note that if this handler is given to a MeowRunner
object, the job_queue_dir will be overwridden by its"""
super().__init__(name=name)
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir
self._print_target, self.debug_level = setup_debugging(print, logging)
print_debug(self._print_target, self.debug_level,
"Created new BashHandler instance", DEBUG_INFO)
def handle(self, event:Dict[str,Any])->None:
"""Function called to handle a given event."""
print_debug(self._print_target, self.debug_level,
f"Handling event {event[EVENT_PATH]}", DEBUG_INFO)
rule = event[EVENT_RULE]
# Assemble job parameters dict from pattern variables
yaml_dict = {}
for var, val in rule.pattern.parameters.items():
yaml_dict[var] = val
for var, val in rule.pattern.outputs.items():
yaml_dict[var] = val
yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH]
# If no parameter sweeps, then one job will suffice
if not rule.pattern.sweep:
self.setup_job(event, yaml_dict)
else:
# If parameter sweeps, then many jobs created
values_list = rule.pattern.expand_sweeps()
for values in values_list:
for value in values:
yaml_dict[value[0]] = value[1]
self.setup_job(event, yaml_dict)
def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an event defintion, if this handler can
process it or not. This handler accepts events from watchdog with
Bash recipes"""
try:
valid_event(event)
msg = ""
if type(event[EVENT_RULE].recipe) != BashRecipe:
msg = "Recipe is not a BashRecipe. "
if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG:
msg += f"Event type is not {EVENT_TYPE_WATCHDOG}."
if msg:
return False, msg
else:
return True, ""
except Exception as e:
return False, str(e)
def _is_valid_job_queue_dir(self, job_queue_dir)->None:
"""Validation check for 'job_queue_dir' variable from main
constructor."""
valid_dir_path(job_queue_dir, must_exist=False)
if not os.path.exists(job_queue_dir):
make_dir(job_queue_dir)
def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None:
"""Function to set up new job dict and send it to the runner to be
executed."""
meow_job = create_job(
JOB_TYPE_BASH,
event,
extras={
JOB_PARAMETERS:yaml_dict,
JOB_HASH: event[WATCHDOG_HASH],
# CONTROL_SCRIPT:python_job_func
}
)
print_debug(self._print_target, self.debug_level,
f"Creating job from event at {event[EVENT_PATH]} of type "
f"{JOB_TYPE_BASH}.", DEBUG_INFO)
# replace MEOW keyworks within variables dict
yaml_dict = replace_keywords(
meow_job[JOB_PARAMETERS],
meow_job[JOB_ID],
event[EVENT_PATH],
event[WATCHDOG_BASE]
)
# Create a base job directory
job_dir = os.path.join(self.job_queue_dir, meow_job[JOB_ID])
make_dir(job_dir)
# write a status file to the job directory
meta_file = os.path.join(job_dir, META_FILE)
write_yaml(meow_job, meta_file)
# parameterise recipe and write as executeable script
base_script = parameterize_bash_script(
event[EVENT_RULE].recipe.recipe, yaml_dict
)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))
write_file(lines_to_string(base_script), base_file)
os.chmod(base_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
# Write job script, to manage base script lifetime and execution
job_script = assemble_bash_job_script()
job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))
write_file(lines_to_string(job_script), job_file)
os.chmod(job_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
meow_job[JOB_STATUS] = STATUS_QUEUED
# update the status file with queued status
write_yaml(meow_job, meta_file)
# Send job directory, as actual definitons will be read from within it
self.to_runner.send(job_dir)
def assemble_bash_job_script()->List[str]:
return [
"#!/bin/bash",
"",
"# Get job params",
"given_hash=$(grep 'file_hash: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)",
"event_path=$(grep 'event_path: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)",
"",
"echo event_path: $event_path",
"echo given_hash: $given_hash",
"",
"# Check hash of input file to avoid race conditions",
"actual_hash=$(sha256sum $event_path | cut -c -64)",
"echo actual_hash: $actual_hash",
"if [ $given_hash != $actual_hash ]; then",
" echo Job was skipped as triggering file has been modified since scheduling",
" exit 134",
"fi",
"",
"# Call actual job script",
"$(dirname $0)/base.sh",
"",
"exit $?"
]

2
recipes/python_recipe.py
View File

@ -65,7 +65,7 @@ class PythonHandler(BaseHandler):
python functions. This does not run as a continuous thread to
handle execution, but is invoked according to a factory pattern using
the handle function. Note that if this handler is given to a MeowRunner
object, the job_queue_dir will be overwridden but its"""
object, the job_queue_dir will be overwridden by its"""
super().__init__(name=name)
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir

3
rules/__init__.py
View File

@ -1,3 +0,0 @@
from .file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from .file_event_python_rule import FileEventPythonRule

43
rules/file_event_jupyter_notebook_rule.py
View File

@ -1,43 +0,0 @@
"""
This file contains definitions for a MEOW rule connecting the FileEventPattern
and JupyterNotebookRecipe.
Author(s): David Marchant
"""
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
# TODO potentailly remove this and just invoke BaseRule directly, as does not
# add any functionality other than some validation.
class FileEventJupyterNotebookRule(BaseRule):
pattern_type = "FileEventPattern"
recipe_type = "JupyterNotebookRecipe"
def __init__(self, name: str, pattern:FileEventPattern,
recipe:JupyterNotebookRecipe):
super().__init__(name, pattern, recipe)
if pattern.recipe != recipe.name:
raise ValueError(f"Cannot create Rule {name}. Pattern "
f"{pattern.name} does not identify Recipe {recipe.name}. It "
f"uses {pattern.recipe}")
def _is_valid_pattern(self, pattern:FileEventPattern)->None:
"""Validation check for 'pattern' variable from main constructor. Is
automatically called during initialisation."""
check_type(
pattern,
FileEventPattern,
hint="FileEventJupyterNotebookRule.pattern"
)
def _is_valid_recipe(self, recipe:JupyterNotebookRecipe)->None:
"""Validation check for 'recipe' variable from main constructor. Is
automatically called during initialisation."""
check_type(
recipe,
JupyterNotebookRecipe,
hint="FileEventJupyterNotebookRule.recipe"
)

39
rules/file_event_python_rule.py
View File

@ -1,39 +0,0 @@
"""
This file contains definitions for a MEOW rule connecting the FileEventPattern
and PythonRecipe.
Author(s): David Marchant
"""
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.python_recipe import PythonRecipe
# TODO potentailly remove this and just invoke BaseRule directly, as does not
# add any functionality other than some validation.
class FileEventPythonRule(BaseRule):
pattern_type = "FileEventPattern"
recipe_type = "PythonRecipe"
def __init__(self, name: str, pattern:FileEventPattern,
recipe:PythonRecipe):
super().__init__(name, pattern, recipe)
def _is_valid_pattern(self, pattern:FileEventPattern)->None:
"""Validation check for 'pattern' variable from main constructor. Is
automatically called during initialisation."""
check_type(
pattern,
FileEventPattern,
hint="FileEventPythonRule.pattern"
)
def _is_valid_recipe(self, recipe:PythonRecipe)->None:
"""Validation check for 'recipe' variable from main constructor. Is
automatically called during initialisation."""
check_type(
recipe,
PythonRecipe,
hint="FileEventPythonRule.recipe"
)

82
tests/shared.py
View File

@ -46,34 +46,36 @@ def backup_before_teardown(backup_source:str, backup_dest:str):
copy_tree(backup_source, backup_dest)
# Recipe funcs
BAREBONES_PYTHON_SCRIPT = [
# Bash scripts
BAREBONES_BASH_SCRIPT = [
""
]
COMPLETE_PYTHON_SCRIPT = [
"import os",
"# Setup parameters",
"num = 1000",
"infile = 'somehere"+ os.path.sep +"particular'",
"outfile = 'nowhere"+ os.path.sep +"particular'",
"",
"with open(infile, 'r') as file:",
" s = float(file.read())",
""
"for i in range(num):",
" s += i",
"",
"div_by = 4",
"result = s / div_by",
"",
"print(result)",
"",
"os.makedirs(os.path.dirname(outfile), exist_ok=True)",
"",
"with open(outfile, 'w') as file:",
" file.write(str(result))",
"",
"print('done')"
COMPLETE_BASH_SCRIPT = [
'#!/bin/bash',
'',
'num=1000',
'infile="data"',
'outfile="output"',
'',
'echo "starting"',
'',
'read var < $infile',
'for (( i=0; i<$num; i++ ))',
'do',
' var=$((var+i))',
'done',
'',
'div_by=4',
'echo $var',
'result=$((var/div_by))',
'',
'echo $result',
'',
'mkdir -p $(dirname $outfile)',
'',
'echo $result > $outfile',
'',
'echo "done"'
]
# Jupyter notebooks
@ -1225,6 +1227,34 @@ GENERATOR_NOTEBOOK = {
}
# Python scripts
BAREBONES_PYTHON_SCRIPT = [
""
]
COMPLETE_PYTHON_SCRIPT = [
"import os",
"# Setup parameters",
"num = 1000",
"infile = 'somehere"+ os.path.sep +"particular'",
"outfile = 'nowhere"+ os.path.sep +"particular'",
"",
"with open(infile, 'r') as file:",
" s = float(file.read())",
""
"for i in range(num):",
" s += i",
"",
"div_by = 4",
"result = s / div_by",
"",
"print(result)",
"",
"os.makedirs(os.path.dirname(outfile), exist_ok=True)",
"",
"with open(outfile, 'w') as file:",
" file.write(str(result))",
"",
"print('done')"
]
IDMC_UTILS_MODULE = [
"import matplotlib.pyplot as plt",
"from sklearn import mixture",

32
tests/test_base.py
View File

@ -8,10 +8,9 @@ from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from meow_base.patterns.file_event_pattern import FileEventPattern
from shared import valid_pattern_one, valid_recipe_one, setup, teardown
from shared import setup, teardown
class BaseRecipeTests(unittest.TestCase):
@ -147,35 +146,6 @@ class BasePatternTests(unittest.TestCase):
self.assertEqual(len(values), 0)
class BaseRuleTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test that BaseRecipe instantiation
def testBaseRule(self)->None:
with self.assertRaises(TypeError):
BaseRule("name", "", "")
class NewRule(BaseRule):
pass
with self.assertRaises(NotImplementedError):
NewRule("name", "", "")
class FullRule(BaseRule):
pattern_type = "pattern"
recipe_type = "recipe"
def _is_valid_recipe(self, recipe:Any)->None:
pass
def _is_valid_pattern(self, pattern:Any)->None:
pass
FullRule("name", valid_pattern_one, valid_recipe_one)
class BaseMonitorTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()

6
tests/test_functionality.py
View File

@ -11,7 +11,7 @@ from sys import prefix, base_prefix
from time import sleep
from typing import Dict
from meow_base.core.base_rule import BaseRule
from meow_base.core.rule import Rule
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
@ -581,7 +581,7 @@ class MeowTests(unittest.TestCase):
def testCreateRule(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
self.assertIsInstance(rule, BaseRule)
self.assertIsInstance(rule, Rule)
with self.assertRaises(ValueError):
rule = create_rule(valid_pattern_one, valid_recipe_two)
@ -607,7 +607,7 @@ class MeowTests(unittest.TestCase):
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
self.assertIsInstance(rule, BaseRule)
self.assertIsInstance(rule, Rule)
self.assertEqual(k, rule.name)
# Test that create_rules creates nothing from invalid pattern inputs

455
tests/test_recipes.py
View File

@ -1,6 +1,8 @@
import jsonschema
import os
import stat
import subprocess
import unittest
from multiprocessing import Pipe
@ -11,22 +13,27 @@ from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, \
EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \
JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \
SWEEP_START, JOB_TYPE_PAPERMILL, get_base_file, get_job_file, \
get_result_file
SWEEP_START, JOB_TYPE_PAPERMILL, JOB_TYPE_BASH, \
get_base_file, get_job_file, get_result_file
from meow_base.core.rule import Rule
from meow_base.functionality.file_io import lines_to_string, make_dir, \
read_yaml, write_file, write_notebook, write_yaml
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_job, create_rules, \
create_rule, create_watchdog_event
from meow_base.functionality.parameterisation import parameterize_bash_script
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.bash_recipe import BashRecipe, BashHandler, \
assemble_bash_job_script
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, papermill_job_func, get_recipe_from_notebook
from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, \
python_job_func
from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule
from shared import BAREBONES_PYTHON_SCRIPT, COMPLETE_PYTHON_SCRIPT, \
TEST_JOB_QUEUE, TEST_MONITOR_BASE, TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, \
APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, setup, teardown
APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, BAREBONES_BASH_SCRIPT, \
COMPLETE_BASH_SCRIPT, \
setup, teardown
class JupyterNotebookTests(unittest.TestCase):
def setUp(self)->None:
@ -157,7 +164,7 @@ class PapermillHandlerTests(unittest.TestCase):
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventJupyterNotebookRule)
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
@ -208,7 +215,7 @@ class PapermillHandlerTests(unittest.TestCase):
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventJupyterNotebookRule)
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
@ -278,7 +285,7 @@ class PapermillHandlerTests(unittest.TestCase):
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventJupyterNotebookRule)
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
@ -576,7 +583,7 @@ class PythonHandlerTests(unittest.TestCase):
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
@ -627,7 +634,7 @@ class PythonHandlerTests(unittest.TestCase):
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
@ -697,7 +704,7 @@ class PythonHandlerTests(unittest.TestCase):
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, FileEventPythonRule)
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
@ -830,6 +837,12 @@ class PythonHandlerTests(unittest.TestCase):
self.assertTrue(os.path.exists(result_path))
with open(result_path, "r") as f:
result = f.read()
self.assertEqual(result, "124937.5")
# Test jobFunc doesn't execute with no args
def testJobFuncBadArgs(self)->None:
try:
@ -878,3 +891,425 @@ class PythonHandlerTests(unittest.TestCase):
EVENT_RULE: rule
})
self.assertTrue(status)
class BashTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test BashRecipe can be created
def testBashRecipeCreationMinimum(self)->None:
BashRecipe("test_recipe", BAREBONES_BASH_SCRIPT)
# Test BashRecipe cannot be created without name
def testBashRecipeCreationNoName(self)->None:
with self.assertRaises(ValueError):
BashRecipe("", BAREBONES_BASH_SCRIPT)
# Test BashRecipe cannot be created with invalid name
def testBashRecipeCreationInvalidName(self)->None:
with self.assertRaises(ValueError):
BashRecipe("@test_recipe", BAREBONES_BASH_SCRIPT)
# Test BashRecipe cannot be created with invalid recipe
def testBashRecipeCreationInvalidRecipe(self)->None:
with self.assertRaises(TypeError):
BashRecipe("test_recipe", BAREBONES_NOTEBOOK)
# Test BashRecipe name setup correctly
def testBashRecipeSetupName(self)->None:
name = "name"
pr = BashRecipe(name, BAREBONES_BASH_SCRIPT)
self.assertEqual(pr.name, name)
# Test BashRecipe recipe setup correctly
def testBashRecipeSetupRecipe(self)->None:
pr = BashRecipe("name", BAREBONES_BASH_SCRIPT)
self.assertEqual(pr.recipe, BAREBONES_BASH_SCRIPT)
# Test BashRecipe parameters setup correctly
def testBashRecipeSetupParameters(self)->None:
parameters = {
"a": 1,
"b": True
}
pr = BashRecipe(
"name", BAREBONES_BASH_SCRIPT, parameters=parameters)
self.assertEqual(pr.parameters, parameters)
# Test BashRecipe requirements setup correctly
def testBashRecipeSetupRequirements(self)->None:
requirements = {
"a": 1,
"b": True
}
pr = BashRecipe(
"name", BAREBONES_BASH_SCRIPT, requirements=requirements)
self.assertEqual(pr.requirements, requirements)
class BashHandlerTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test BashHandler can be created
def testBashHandlerMinimum(self)->None:
BashHandler(job_queue_dir=TEST_JOB_QUEUE)
# Test BashHandler naming
def testBashHandlerNaming(self)->None:
test_name = "test_name"
handler = BashHandler(name=test_name)
self.assertEqual(handler.name, test_name)
handler = BashHandler()
self.assertTrue(handler.name.startswith("handler_"))
# Test BashHandler will handle given events
def testBashHandlerHandling(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = BashRecipe(
"recipe_one", COMPLETE_BASH_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
if from_handler_reader.poll(3):
job_dir = from_handler_reader.recv()
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
# Test BashHandler will create enough jobs from single sweep
def testBashHandlerHandlingSingleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={"s":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
}})
recipe = BashRecipe(
"recipe_one", COMPLETE_BASH_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
else:
recieving = False
values = [0, 1, 2]
self.assertEqual(len(jobs), 3)
for job_dir in jobs:
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
self.assertIn(JOB_PARAMETERS, job)
self.assertIn("s", job[JOB_PARAMETERS])
if job[JOB_PARAMETERS]["s"] in values:
values.remove(job[JOB_PARAMETERS]["s"])
self.assertEqual(len(values), 0)
# Test BashHandler will create enough jobs from multiple sweeps
def testBashHandlerHandlingMultipleSweep(self)->None:
from_handler_reader, from_handler_writer = Pipe()
ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE)
ph.to_runner = from_handler_writer
with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f:
f.write("Data")
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one", sweep={
"s1":{
SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1
},
"s2":{
SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15
}
})
recipe = BashRecipe(
"recipe_one", COMPLETE_BASH_SCRIPT)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
self.assertEqual(len(rules), 1)
_, rule = rules.popitem()
self.assertIsInstance(rule, Rule)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
event = {
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"),
WATCHDOG_BASE: TEST_MONITOR_BASE,
EVENT_RULE: rule,
WATCHDOG_HASH: get_file_hash(
os.path.join(TEST_MONITOR_BASE, "A"), SHA256
)
}
ph.handle(event)
jobs = []
recieving = True
while recieving:
if from_handler_reader.poll(3):
jobs.append(from_handler_reader.recv())
else:
recieving = False
values = [
"s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20",
"s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35",
"s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50",
"s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65",
"s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80",
]
self.assertEqual(len(jobs), 15)
for job_dir in jobs:
self.assertIsInstance(job_dir, str)
self.assertTrue(os.path.exists(job_dir))
job = read_yaml(os.path.join(job_dir, META_FILE))
valid_job(job)
self.assertIn(JOB_PARAMETERS, job)
val1 = None
val2 = None
if "s1" in job[JOB_PARAMETERS]:
val1 = f"s1-{job[JOB_PARAMETERS]['s1']}"
if "s2" in job[JOB_PARAMETERS]:
val2 = f"s2-{job[JOB_PARAMETERS]['s2']}"
val = None
if val1 and val2:
val = f"{val1}/{val2}"
if val and val in values:
values.remove(val)
self.assertEqual(len(values), 0)
# Test jobFunc performs as expected
def testJobFunc(self)->None:
file_path = os.path.join(TEST_MONITOR_BASE, "test")
result_path = os.path.join(TEST_MONITOR_BASE, "output")
with open(file_path, "w") as f:
f.write("250")
file_hash = get_file_hash(file_path, SHA256)
pattern = FileEventPattern(
"pattern",
file_path,
"recipe_one",
"infile",
parameters={
"extra":"A line from a test Pattern",
"outfile": result_path
})
recipe = BashRecipe(
"recipe_one", COMPLETE_BASH_SCRIPT)
rule = create_rule(pattern, recipe)
params_dict = {
"extra":"extra",
"infile":file_path,
"outfile": result_path
}
job_dict = create_job(
JOB_TYPE_BASH,
create_watchdog_event(
file_path,
rule,
TEST_MONITOR_BASE,
file_hash
),
extras={
JOB_PARAMETERS:params_dict,
JOB_HASH: file_hash
}
)
job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID])
make_dir(job_dir)
meta_file = os.path.join(job_dir, META_FILE)
write_yaml(job_dict, meta_file)
base_script = parameterize_bash_script(
COMPLETE_BASH_SCRIPT, params_dict
)
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))
write_file(lines_to_string(base_script), base_file)
st = os.stat(base_file)
os.chmod(base_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
job_script = assemble_bash_job_script()
job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))
write_file(lines_to_string(job_script), job_file)
st = os.stat(job_file)
os.chmod(job_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
print(os.listdir(job_dir))
print(os.getcwd())
result = subprocess.call(job_file, cwd=".")
self.assertEqual(result, 0)
self.assertTrue(os.path.exists(job_dir))
meta_path = os.path.join(job_dir, META_FILE)
self.assertTrue(os.path.exists(meta_path))
status = read_yaml(meta_path)
self.assertIsInstance(status, Dict)
self.assertIn(JOB_STATUS, status)
self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS])
self.assertNotIn(JOB_ERROR, status)
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_base_file(JOB_TYPE_BASH))))
self.assertTrue(os.path.exists(
os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))))
self.assertTrue(os.path.exists(result_path))
with open(result_path, "r") as f:
result = f.read()
self.assertEqual(result, "124937\n")
# Test jobFunc doesn't execute with no args
def testJobFuncBadArgs(self)->None:
try:
Bash_job_func({})
except Exception:
pass
self.assertEqual(len(os.listdir(TEST_JOB_QUEUE)), 0)
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0)
# Test handling criteria function
def testValidHandleCriteria(self)->None:
ph = BashHandler()
pattern = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = BashRecipe(
"recipe_one", BAREBONES_BASH_SCRIPT
)
rule = create_rule(pattern, recipe)
status, _ = ph.valid_handle_criteria({})
self.assertFalse(status)
status, _ = ph.valid_handle_criteria("")
self.assertFalse(status)
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: "type",
EVENT_RULE: rule
})
self.assertFalse(status)
status, _ = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: "rule"
})
self.assertFalse(status)
status, s = ph.valid_handle_criteria({
EVENT_PATH: "path",
EVENT_TYPE: EVENT_TYPE_WATCHDOG,
EVENT_RULE: rule
})
self.assertTrue(status)

92
tests/test_rule.py Normal file
View File

@ -0,0 +1,92 @@
import unittest
from meow_base.core.rule import Rule
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import BAREBONES_NOTEBOOK, setup, teardown
class CorrectnessTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test Rule created from valid pattern and recipe
def testRuleCreationMinimum(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
Rule(fep, jnr)
# Test Rule not created with empty name
def testRuleCreationNoName(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
r = Rule(fep, jnr)
self.assertIsInstance(r.name, str)
self.assertTrue(len(r.name) > 1)
# Test Rule not created with invalid name
def testRuleCreationInvalidName(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
with self.assertRaises(TypeError):
Rule(fep, jnr, name=1)
# Test Rule not created with invalid pattern
def testRuleCreationInvalidPattern(self)->None:
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
with self.assertRaises(TypeError):
Rule("pattern", jnr)
# Test Rule not created with invalid recipe
def testRuleCreationInvalidRecipe(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
with self.assertRaises(TypeError):
Rule(fep, "recipe")
# Test Rule not created with mismatched recipe
def testRuleCreationMissmatchedRecipe(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
with self.assertRaises(ValueError):
Rule(fep, jnr)
# Test Rule created with valid name
def testRuleSetupName(self)->None:
name = "name"
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
fejnr = Rule(fep, jnr, name=name)
self.assertEqual(fejnr.name, name)
# Test Rule not created with valid pattern
def testRuleSetupPattern(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
fejnr = Rule(fep, jnr)
self.assertEqual(fejnr.pattern, fep)
# Test Rule not created with valid recipe
def testRuleSetupRecipe(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
fejnr = Rule(fep, jnr)
self.assertEqual(fejnr.recipe, jnr)

91
tests/test_rules.py
View File

@ -1,91 +0,0 @@
import unittest
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from meow_base.rules.file_event_jupyter_notebook_rule import \
FileEventJupyterNotebookRule
from shared import BAREBONES_NOTEBOOK, setup, teardown
class CorrectnessTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
setup()
def tearDown(self)->None:
super().tearDown()
teardown()
# Test FileEventJupyterNotebookRule created from valid pattern and recipe
def testFileEventJupyterNotebookRuleCreationMinimum(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
FileEventJupyterNotebookRule("name", fep, jnr)
# Test FileEventJupyterNotebookRule not created with empty name
def testFileEventJupyterNotebookRuleCreationNoName(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
with self.assertRaises(ValueError):
FileEventJupyterNotebookRule("", fep, jnr)
# Test FileEventJupyterNotebookRule not created with invalid name
def testFileEventJupyterNotebookRuleCreationInvalidName(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
with self.assertRaises(TypeError):
FileEventJupyterNotebookRule(1, fep, jnr)
# Test FileEventJupyterNotebookRule not created with invalid pattern
def testFileEventJupyterNotebookRuleCreationInvalidPattern(self)->None:
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
with self.assertRaises(TypeError):
FileEventJupyterNotebookRule("name", "pattern", jnr)
# Test FileEventJupyterNotebookRule not created with invalid recipe
def testFileEventJupyterNotebookRuleCreationInvalidRecipe(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
with self.assertRaises(TypeError):
FileEventJupyterNotebookRule("name", fep, "recipe")
# Test FileEventJupyterNotebookRule not created with mismatched recipe
def testFileEventJupyterNotebookRuleCreationMissmatchedRecipe(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK)
with self.assertRaises(ValueError):
FileEventJupyterNotebookRule("name", fep, jnr)
# Test FileEventJupyterNotebookRule created with valid name
def testFileEventJupyterNotebookRuleSetupName(self)->None:
name = "name"
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
fejnr = FileEventJupyterNotebookRule(name, fep, jnr)
self.assertEqual(fejnr.name, name)
# Test FileEventJupyterNotebookRule not created with valid pattern
def testFileEventJupyterNotebookRuleSetupPattern(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
fejnr = FileEventJupyterNotebookRule("name", fep, jnr)
self.assertEqual(fejnr.pattern, fep)
# Test FileEventJupyterNotebookRule not created with valid recipe
def testFileEventJupyterNotebookRuleSetupRecipe(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK)
fejnr = FileEventJupyterNotebookRule("name", fep, jnr)
self.assertEqual(fejnr.recipe, jnr)