diff --git a/conductors/local_bash_conductor.py b/conductors/local_bash_conductor.py new file mode 100644 index 0000000..6ebcc89 --- /dev/null +++ b/conductors/local_bash_conductor.py @@ -0,0 +1,129 @@ + +""" +This file contains definitions for the LocalBashConductor, in order to +execute Bash jobs on the local resource. + +Author(s): David Marchant +""" + +import os +import shutil +import subprocess + +from datetime import datetime +from typing import Any, Dict, Tuple + +from meow_base.core.base_conductor import BaseConductor +from meow_base.core.correctness.meow import valid_job +from meow_base.core.correctness.vars import DEFAULT_JOB_QUEUE_DIR, \ + DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, META_FILE, JOB_STATUS, \ + BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \ + JOB_ERROR, JOB_TYPE, DEFAULT_JOB_QUEUE_DIR, STATUS_RUNNING, \ + JOB_START_TIME, DEFAULT_JOB_OUTPUT_DIR, get_job_file +from meow_base.core.correctness.validation import valid_dir_path +from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \ + write_yaml + + +class LocalBashConductor(BaseConductor): + def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, + job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR, name:str="")->None: + """LocalBashConductor Constructor. This should be used to execute + Bash jobs, and will then pass any internal job runner files to the + output directory. Note that if this handler is given to a MeowRunner + object, the job_queue_dir and job_output_dir will be overwridden.""" + super().__init__(name=name) + self._is_valid_job_queue_dir(job_queue_dir) + self.job_queue_dir = job_queue_dir + self._is_valid_job_output_dir(job_output_dir) + self.job_output_dir = job_output_dir + + def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: + """Function to determine given an job defintion, if this conductor can + process it or not. This conductor will accept any Bash job type""" + try: + valid_job(job) + msg = "" + if job[JOB_TYPE] not in [JOB_TYPE_BASH]: + msg = f"Job type was not {JOB_TYPE_BASH}." + if msg: + return False, msg + else: + return True, "" + except Exception as e: + return False, str(e) + + def execute(self, job_dir:str)->None: + """Function to actually execute a Bash job. This will read job + defintions from its meta file, update the meta file and attempt to + execute. Some unspecific feedback will be given on execution failure, + but depending on what it is it may be up to the job itself to provide + more detailed feedback.""" + valid_dir_path(job_dir, must_exist=True) + + # Test our job parameters. Even if its gibberish, we still move to + # output + abort = False + try: + meta_file = os.path.join(job_dir, META_FILE) + job = read_yaml(meta_file) + valid_job(job) + + # update the status file with running status + job[JOB_STATUS] = STATUS_RUNNING + job[JOB_START_TIME] = datetime.now() + write_yaml(job, meta_file) + + except Exception as e: + # If something has gone wrong at this stage then its bad, so we + # need to make our own error file + error_file = os.path.join(job_dir, BACKUP_JOB_ERROR_FILE) + write_file(f"Recieved incorrectly setup job.\n\n{e}", error_file) + abort = True + + # execute the job + if not abort: + try: + result = subprocess.call(get_job_file(JOB_TYPE_BASH), cwd=".") + + # get up to date job data + job = read_yaml(meta_file) + + # Update the status file with the finalised status + job[JOB_STATUS] = STATUS_DONE + job[JOB_END_TIME] = datetime.now() + write_yaml(job, meta_file) + + except Exception as e: + # get up to date job data + job = read_yaml(meta_file) + + # Update the status file with the error status. Don't overwrite + # any more specific error messages already created + if JOB_STATUS not in job: + job[JOB_STATUS] = STATUS_FAILED + if JOB_END_TIME not in job: + job[JOB_END_TIME] = datetime.now() + if JOB_ERROR not in job: + job[JOB_ERROR] = f"Job execution failed. {e}" + write_yaml(job, meta_file) + + # Move the contents of the execution directory to the final output + # directory. + job_output_dir = \ + os.path.join(self.job_output_dir, os.path.basename(job_dir)) + shutil.move(job_dir, job_output_dir) + + def _is_valid_job_queue_dir(self, job_queue_dir)->None: + """Validation check for 'job_queue_dir' variable from main + constructor.""" + valid_dir_path(job_queue_dir, must_exist=False) + if not os.path.exists(job_queue_dir): + make_dir(job_queue_dir) + + def _is_valid_job_output_dir(self, job_output_dir)->None: + """Validation check for 'job_output_dir' variable from main + constructor.""" + valid_dir_path(job_output_dir, must_exist=False) + if not os.path.exists(job_output_dir): + make_dir(job_output_dir) diff --git a/core/base_monitor.py b/core/base_monitor.py index 7533694..227d676 100644 --- a/core/base_monitor.py +++ b/core/base_monitor.py @@ -11,7 +11,7 @@ from typing import Union, Dict from meow_base.core.base_pattern import BasePattern from meow_base.core.base_recipe import BaseRecipe -from meow_base.core.base_rule import BaseRule +from meow_base.core.rule import Rule from meow_base.core.correctness.vars import VALID_CHANNELS, \ VALID_MONITOR_NAME_CHARS, get_drt_imp_msg from meow_base.core.correctness.validation import check_implementation, \ @@ -29,7 +29,7 @@ class BaseMonitor: # A collection of recipes _recipes: Dict[str, BaseRecipe] # A collection of rules derived from _patterns and _recipes - _rules: Dict[str, BaseRule] + _rules: Dict[str, Rule] # A channel for sending messages to the runner. Note that this is not # initialised within the constructor, but within the runner when passed the # monitor is passed to it. @@ -138,7 +138,7 @@ class BaseMonitor: Must be implemented by any child process.""" pass - def get_rules(self)->Dict[str,BaseRule]: + def get_rules(self)->Dict[str,Rule]: """Function to get a dictionary of all current rule definitions. Must be implemented by any child process.""" pass diff --git a/core/base_rule.py b/core/base_rule.py deleted file mode 100644 index 7b44227..0000000 --- a/core/base_rule.py +++ /dev/null @@ -1,85 +0,0 @@ - -""" -This file contains the base MEOW rule defintion. This should be inherited from -for all rule instances. - -Author(s): David Marchant -""" - -from sys import modules -from typing import Any - -if "BasePattern" not in modules: - from meow_base.core.base_pattern import BasePattern -if "BaseRecipe" not in modules: - from meow_base.core.base_recipe import BaseRecipe -from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \ - get_drt_imp_msg -from meow_base.core.correctness.validation import valid_string, check_type, \ - check_implementation - - -class BaseRule: - # A unique identifier for the rule - name:str - # A pattern to be used in rule triggering - pattern:BasePattern - # A recipe to be used in rule execution - recipe:BaseRecipe - # The string name of the pattern class that can be used to create this rule - pattern_type:str="" - # The string name of the recipe class that can be used to create this rule - recipe_type:str="" - def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe): - """BaseRule Constructor. This will check that any class inheriting - from it implements its validation functions. It will then call these on - the input parameters.""" - check_implementation(type(self)._is_valid_pattern, BaseRule) - check_implementation(type(self)._is_valid_recipe, BaseRule) - self.__check_types_set() - self._is_valid_name(name) - self.name = name - self._is_valid_pattern(pattern) - self.pattern = pattern - self._is_valid_recipe(recipe) - self.recipe = recipe - check_type(pattern, BasePattern, hint="BaseRule.pattern") - check_type(recipe, BaseRecipe, hint="BaseRule.recipe") - if pattern.recipe != recipe.name: - raise ValueError(f"Cannot create Rule {name}. Pattern " - f"{pattern.name} does not identify Recipe {recipe.name}. It " - f"uses {pattern.recipe}") - - def __new__(cls, *args, **kwargs): - """A check that this base class is not instantiated itself, only - inherited from""" - if cls is BaseRule: - msg = get_drt_imp_msg(BaseRule) - raise TypeError(msg) - return object.__new__(cls) - - def _is_valid_name(self, name:str)->None: - """Validation check for 'name' variable from main constructor. Is - automatically called during initialisation. This does not need to be - overridden by child classes.""" - valid_string(name, VALID_RULE_NAME_CHARS) - - def _is_valid_pattern(self, pattern:Any)->None: - """Validation check for 'pattern' variable from main constructor. Must - be implemented by any child class.""" - pass - - def _is_valid_recipe(self, recipe:Any)->None: - """Validation check for 'recipe' variable from main constructor. Must - be implemented by any child class.""" - pass - - def __check_types_set(self)->None: - """Validation check that the self.pattern_type and self.recipe_type - attributes have been set in a child class.""" - if self.pattern_type == "": - raise AttributeError(f"Rule Class '{self.__class__.__name__}' " - "does not set a pattern_type.") - if self.recipe_type == "": - raise AttributeError(f"Rule Class '{self.__class__.__name__}' " - "does not set a recipe_type.") diff --git a/core/correctness/meow.py b/core/correctness/meow.py index 28efe9d..a2f6fa4 100644 --- a/core/correctness/meow.py +++ b/core/correctness/meow.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Any, Dict, Type -from meow_base.core.base_rule import BaseRule +from meow_base.core.rule import Rule from meow_base.core.correctness.validation import check_type from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \ JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \ @@ -13,7 +13,7 @@ EVENT_KEYS = { EVENT_TYPE: str, EVENT_PATH: str, # Should be a Rule but can't import here due to circular dependencies - EVENT_RULE: BaseRule + EVENT_RULE: Rule } WATCHDOG_EVENT_KEYS = { diff --git a/core/correctness/vars.py b/core/correctness/vars.py index 205ba13..a078f2f 100644 --- a/core/correctness/vars.py +++ b/core/correctness/vars.py @@ -86,6 +86,7 @@ DEFAULT_JOB_OUTPUT_DIR = "job_output" # meow jobs JOB_TYPE = "job_type" +JOB_TYPE_BASH = "bash" JOB_TYPE_PYTHON = "python" JOB_TYPE_PAPERMILL = "papermill" PYTHON_FUNC = "func" @@ -94,12 +95,17 @@ JOB_TYPES = { JOB_TYPE_PAPERMILL: [ "base.ipynb", "job.ipynb", - "result.ipynb", + "result.ipynb" ], JOB_TYPE_PYTHON: [ "base.py", "job.py", - "result.py", + "result.py" + ], + JOB_TYPE_BASH: [ + "base.sh", + "job.sh", + "result.sh" ] } diff --git a/core/rule.py b/core/rule.py new file mode 100644 index 0000000..22fc99a --- /dev/null +++ b/core/rule.py @@ -0,0 +1,49 @@ + +""" +This file contains the MEOW rule defintion. + +Author(s): David Marchant +""" + +from sys import modules +from typing import Any + +if "BasePattern" not in modules: + from meow_base.core.base_pattern import BasePattern +if "BaseRecipe" not in modules: + from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \ + get_drt_imp_msg +from meow_base.core.correctness.validation import valid_string, check_type, \ + check_implementation +from meow_base.functionality.naming import generate_rule_id + + +class Rule: + # A unique identifier for the rule + name:str + # A pattern to be used in rule triggering + pattern:BasePattern + # A recipe to be used in rule execution + recipe:BaseRecipe + def __init__(self, pattern:BasePattern, recipe:BaseRecipe, name:str=""): + """Rule Constructor. This will check that any class inheriting + from it implements its validation functions. It will then call these on + the input parameters.""" + if not name: + name = generate_rule_id() + self._is_valid_name(name) + self.name = name + check_type(pattern, BasePattern, hint="Rule.pattern") + self.pattern = pattern + check_type(recipe, BaseRecipe, hint="Rule.recipe") + self.recipe = recipe + if pattern.recipe != recipe.name: + raise ValueError(f"Cannot create Rule {name}. Pattern " + f"{pattern.name} does not identify Recipe {recipe.name}. It " + f"uses {pattern.recipe}") + + def _is_valid_name(self, name:str)->None: + """Validation check for 'name' variable from main constructor. Is + automatically called during initialisation.""" + valid_string(name, VALID_RULE_NAME_CHARS) diff --git a/functionality/meow.py b/functionality/meow.py index 21f69de..9fadd2c 100644 --- a/functionality/meow.py +++ b/functionality/meow.py @@ -10,7 +10,7 @@ from typing import Any, Dict, Union, List from meow_base.core.base_pattern import BasePattern from meow_base.core.base_recipe import BaseRecipe -from meow_base.core.base_rule import BaseRule +from meow_base.core.rule import Rule from meow_base.core.correctness.validation import check_type, valid_dict, \ valid_list from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \ @@ -18,7 +18,7 @@ from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \ JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \ JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \ SWEEP_START, SWEEP_STOP -from meow_base.functionality.naming import generate_job_id, generate_rule_id +from meow_base.functionality.naming import generate_job_id # mig trigger keyword replacements KEYWORD_PATH = "{PATH}" @@ -147,8 +147,7 @@ def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={} return {**extras, **job_dict} def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], - recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]], - new_rules:List[BaseRule]=[])->Dict[str,BaseRule]: + recipes:Union[Dict[str,BaseRecipe],List[BaseRecipe]])->Dict[str,Rule]: """Function to create any valid rules from a given collection of patterns and recipes. All inbuilt rule types are considered, with additional definitions provided through the 'new_rules' variable. Note that any @@ -157,7 +156,6 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], # Validation of inputs check_type(patterns, Dict, alt_types=[List], hint="create_rules.patterns") check_type(recipes, Dict, alt_types=[List], hint="create_rules.recipes") - valid_list(new_rules, BaseRule, min_length=0) # Convert a pattern list to a dictionary if isinstance(patterns, list): @@ -198,34 +196,14 @@ def create_rules(patterns:Union[Dict[str,BasePattern],List[BasePattern]], pass return generated_rules -def create_rule(pattern:BasePattern, recipe:BaseRecipe, - new_rules:List[BaseRule]=[])->BaseRule: +def create_rule(pattern:BasePattern, recipe:BaseRecipe)->Rule: """Function to create a valid rule from a given pattern and recipe. All inbuilt rule types are considered, with additional definitions provided through the 'new_rules' variable.""" check_type(pattern, BasePattern, hint="create_rule.pattern") check_type(recipe, BaseRecipe, hint="create_rule.recipe") - valid_list(new_rules, BaseRule, min_length=0, hint="create_rule.new_rules") - - # TODO fix me - # Imported here to avoid circular imports at top of file - import meow_base.rules - all_rules = { - (r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__() - } - - # Add in new rules - for rule in new_rules: - all_rules[(rule.pattern_type, rule.recipe_type)] = rule - - # Find appropriate rule type from pattern and recipe types - key = (type(pattern).__name__, type(recipe).__name__) - if (key) in all_rules: - return all_rules[key]( - generate_rule_id(), - pattern, - recipe - ) - # Raise error if not valid rule type can be found - raise TypeError(f"No valid rule for Pattern '{pattern}' and Recipe " - f"'{recipe}' could be found.") + + return Rule( + pattern, + recipe + ) diff --git a/functionality/parameterisation.py b/functionality/parameterisation.py index 4c4e42d..c0640ad 100644 --- a/functionality/parameterisation.py +++ b/functionality/parameterisation.py @@ -119,3 +119,36 @@ def parameterize_python_script(script:List[str], parameters:Dict[str,Any], check_script(output_script) return output_script + +def parameterize_bash_script(script:List[str], parameters:Dict[str,Any], + expand_env_values:bool=False)->Dict[str,Any]: + check_script(script) + check_type(parameters, Dict + ,hint="parameterize_bash_script.parameters") + + output_script = deepcopy(script) + + for i, line in enumerate(output_script): + if "=" in line: + d_line = list(map(lambda x: x.replace(" ", ""), + line.split("="))) + # Matching parameter name + if len(d_line) == 2 and d_line[0] in parameters: + value = parameters[d_line[0]] + # Whether to expand value from os env + if ( + expand_env_values + and isinstance(value, str) + and value.startswith("ENV_") + ): + env_var = value.replace("ENV_", "") + value = getenv( + env_var, + "MISSING ENVIRONMENT VARIABLE: {}".format(env_var) + ) + output_script[i] = f"{d_line[0]}={repr(value)}" + + # Validate that the parameterized notebook is still valid + check_script(output_script) + + return output_script \ No newline at end of file diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 42756d8..df95371 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -21,7 +21,7 @@ from watchdog.events import PatternMatchingEventHandler from meow_base.core.base_recipe import BaseRecipe from meow_base.core.base_monitor import BaseMonitor from meow_base.core.base_pattern import BasePattern -from meow_base.core.base_rule import BaseRule +from meow_base.core.rule import Rule from meow_base.core.correctness.validation import check_type, valid_string, \ valid_dict, valid_list, valid_dir_path from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ @@ -358,7 +358,7 @@ class WatchdogMonitor(BaseMonitor): self._recipes_lock.release() return to_return - def get_rules(self)->Dict[str,BaseRule]: + def get_rules(self)->Dict[str,Rule]: """Function to get a dict of the currently defined rules of the monitor. Note that the result is deep-copied, and so can be manipulated without directly manipulating the internals of the monitor.""" @@ -486,7 +486,7 @@ class WatchdogMonitor(BaseMonitor): for rule in self._rules.values(): self._apply_retroactive_rule(rule) - def _apply_retroactive_rule(self, rule:BaseRule)->None: + def _apply_retroactive_rule(self, rule:Rule)->None: """Function to determine if a rule should be applied to the existing file structure, were the file structure created/modified now.""" self._rules_lock.acquire() diff --git a/recipes/bash_recipe.py b/recipes/bash_recipe.py new file mode 100644 index 0000000..971ad61 --- /dev/null +++ b/recipes/bash_recipe.py @@ -0,0 +1,212 @@ + +import os +import stat +import sys + +from typing import Any, Dict, List, Tuple + +from meow_base.core.base_handler import BaseHandler +from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.correctness.meow import valid_event +from meow_base.core.correctness.validation import check_type, valid_dict, \ + valid_string, valid_dir_path +from meow_base.core.correctness.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \ + VALID_VARIABLE_NAME_CHARS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, JOB_ID, \ + EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_PARAMETERS, JOB_HASH, \ + WATCHDOG_HASH, WATCHDOG_BASE, META_FILE, PARAMS_FILE, STATUS_QUEUED, \ + JOB_STATUS, \ + get_base_file, get_job_file +from meow_base.functionality.debug import setup_debugging, print_debug +from meow_base.functionality.file_io import valid_path, make_dir, write_yaml, \ + write_file, lines_to_string +from meow_base.functionality.parameterisation import parameterize_bash_script +from meow_base.functionality.meow import create_job, replace_keywords + + +class BashRecipe(BaseRecipe): + # A path to the bash script used to create this recipe + def __init__(self, name:str, recipe:Any, parameters:Dict[str,Any]={}, + requirements:Dict[str,Any]={}, source:str=""): + """BashRecipe Constructor. This is used to execute bash scripts, + enabling anything not natively supported by MEOW.""" + super().__init__(name, recipe, parameters, requirements) + self._is_valid_source(source) + self.source = source + + def _is_valid_source(self, source:str)->None: + """Validation check for 'source' variable from main constructor.""" + if source: + valid_path(source, extension=".sh", min_length=0) + + def _is_valid_recipe(self, recipe:List[str])->None: + """Validation check for 'recipe' variable from main constructor. + Called within parent BaseRecipe constructor.""" + check_type(recipe, List, hint="BashRecipe.recipe") + for line in recipe: + check_type(line, str, hint="BashRecipe.recipe[line]") + + def _is_valid_parameters(self, parameters:Dict[str,Any])->None: + """Validation check for 'parameters' variable from main constructor. + Called within parent BaseRecipe constructor.""" + valid_dict(parameters, str, Any, strict=False, min_length=0) + for k in parameters.keys(): + valid_string(k, VALID_VARIABLE_NAME_CHARS) + + def _is_valid_requirements(self, requirements:Dict[str,Any])->None: + """Validation check for 'requirements' variable from main constructor. + Called within parent BaseRecipe constructor.""" + valid_dict(requirements, str, Any, strict=False, min_length=0) + for k in requirements.keys(): + valid_string(k, VALID_VARIABLE_NAME_CHARS) + + +class BashHandler(BaseHandler): + # Config option, above which debug messages are ignored + debug_level:int + # Where print messages are sent + _print_target:Any + def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, name:str="", + print:Any=sys.stdout, logging:int=0)->None: + """BashHandler Constructor. This creates jobs to be executed as + bash scripts. This does not run as a continuous thread to + handle execution, but is invoked according to a factory pattern using + the handle function. Note that if this handler is given to a MeowRunner + object, the job_queue_dir will be overwridden by its""" + super().__init__(name=name) + self._is_valid_job_queue_dir(job_queue_dir) + self.job_queue_dir = job_queue_dir + self._print_target, self.debug_level = setup_debugging(print, logging) + print_debug(self._print_target, self.debug_level, + "Created new BashHandler instance", DEBUG_INFO) + + def handle(self, event:Dict[str,Any])->None: + """Function called to handle a given event.""" + print_debug(self._print_target, self.debug_level, + f"Handling event {event[EVENT_PATH]}", DEBUG_INFO) + + rule = event[EVENT_RULE] + + # Assemble job parameters dict from pattern variables + yaml_dict = {} + for var, val in rule.pattern.parameters.items(): + yaml_dict[var] = val + for var, val in rule.pattern.outputs.items(): + yaml_dict[var] = val + yaml_dict[rule.pattern.triggering_file] = event[EVENT_PATH] + + # If no parameter sweeps, then one job will suffice + if not rule.pattern.sweep: + self.setup_job(event, yaml_dict) + else: + # If parameter sweeps, then many jobs created + values_list = rule.pattern.expand_sweeps() + for values in values_list: + for value in values: + yaml_dict[value[0]] = value[1] + self.setup_job(event, yaml_dict) + + def valid_handle_criteria(self, event:Dict[str,Any])->Tuple[bool,str]: + """Function to determine given an event defintion, if this handler can + process it or not. This handler accepts events from watchdog with + Bash recipes""" + try: + valid_event(event) + msg = "" + if type(event[EVENT_RULE].recipe) != BashRecipe: + msg = "Recipe is not a BashRecipe. " + if event[EVENT_TYPE] != EVENT_TYPE_WATCHDOG: + msg += f"Event type is not {EVENT_TYPE_WATCHDOG}." + if msg: + return False, msg + else: + return True, "" + except Exception as e: + return False, str(e) + + def _is_valid_job_queue_dir(self, job_queue_dir)->None: + """Validation check for 'job_queue_dir' variable from main + constructor.""" + valid_dir_path(job_queue_dir, must_exist=False) + if not os.path.exists(job_queue_dir): + make_dir(job_queue_dir) + + def setup_job(self, event:Dict[str,Any], yaml_dict:Dict[str,Any])->None: + """Function to set up new job dict and send it to the runner to be + executed.""" + meow_job = create_job( + JOB_TYPE_BASH, + event, + extras={ + JOB_PARAMETERS:yaml_dict, + JOB_HASH: event[WATCHDOG_HASH], +# CONTROL_SCRIPT:python_job_func + } + ) + print_debug(self._print_target, self.debug_level, + f"Creating job from event at {event[EVENT_PATH]} of type " + f"{JOB_TYPE_BASH}.", DEBUG_INFO) + + # replace MEOW keyworks within variables dict + yaml_dict = replace_keywords( + meow_job[JOB_PARAMETERS], + meow_job[JOB_ID], + event[EVENT_PATH], + event[WATCHDOG_BASE] + ) + + # Create a base job directory + job_dir = os.path.join(self.job_queue_dir, meow_job[JOB_ID]) + make_dir(job_dir) + + # write a status file to the job directory + meta_file = os.path.join(job_dir, META_FILE) + write_yaml(meow_job, meta_file) + + # parameterise recipe and write as executeable script + base_script = parameterize_bash_script( + event[EVENT_RULE].recipe.recipe, yaml_dict + ) + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH)) + write_file(lines_to_string(base_script), base_file) + os.chmod(base_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + # Write job script, to manage base script lifetime and execution + + job_script = assemble_bash_job_script() + job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH)) + write_file(lines_to_string(job_script), job_file) + os.chmod(job_file, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + meow_job[JOB_STATUS] = STATUS_QUEUED + + # update the status file with queued status + write_yaml(meow_job, meta_file) + + # Send job directory, as actual definitons will be read from within it + self.to_runner.send(job_dir) + + +def assemble_bash_job_script()->List[str]: + return [ + "#!/bin/bash", + "", + "# Get job params", + "given_hash=$(grep 'file_hash: *' $(dirname $0)/job.yml | tail -n1 | cut -c 14-)", + "event_path=$(grep 'event_path: *' $(dirname $0)/job.yml | tail -n1 | cut -c 15-)", + "", + "echo event_path: $event_path", + "echo given_hash: $given_hash", + "", + "# Check hash of input file to avoid race conditions", + "actual_hash=$(sha256sum $event_path | cut -c -64)", + "echo actual_hash: $actual_hash", + "if [ $given_hash != $actual_hash ]; then", + " echo Job was skipped as triggering file has been modified since scheduling", + " exit 134", + "fi", + "", + "# Call actual job script", + "$(dirname $0)/base.sh", + "", + "exit $?" + ] diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index 1630e32..4db6dd8 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -65,7 +65,7 @@ class PythonHandler(BaseHandler): python functions. This does not run as a continuous thread to handle execution, but is invoked according to a factory pattern using the handle function. Note that if this handler is given to a MeowRunner - object, the job_queue_dir will be overwridden but its""" + object, the job_queue_dir will be overwridden by its""" super().__init__(name=name) self._is_valid_job_queue_dir(job_queue_dir) self.job_queue_dir = job_queue_dir diff --git a/rules/__init__.py b/rules/__init__.py deleted file mode 100644 index 6a5156c..0000000 --- a/rules/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ - -from .file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule -from .file_event_python_rule import FileEventPythonRule \ No newline at end of file diff --git a/rules/file_event_jupyter_notebook_rule.py b/rules/file_event_jupyter_notebook_rule.py deleted file mode 100644 index de1af8d..0000000 --- a/rules/file_event_jupyter_notebook_rule.py +++ /dev/null @@ -1,43 +0,0 @@ - -""" -This file contains definitions for a MEOW rule connecting the FileEventPattern -and JupyterNotebookRecipe. - -Author(s): David Marchant -""" - -from meow_base.core.base_rule import BaseRule -from meow_base.core.correctness.validation import check_type -from meow_base.patterns.file_event_pattern import FileEventPattern -from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe - -# TODO potentailly remove this and just invoke BaseRule directly, as does not -# add any functionality other than some validation. -class FileEventJupyterNotebookRule(BaseRule): - pattern_type = "FileEventPattern" - recipe_type = "JupyterNotebookRecipe" - def __init__(self, name: str, pattern:FileEventPattern, - recipe:JupyterNotebookRecipe): - super().__init__(name, pattern, recipe) - if pattern.recipe != recipe.name: - raise ValueError(f"Cannot create Rule {name}. Pattern " - f"{pattern.name} does not identify Recipe {recipe.name}. It " - f"uses {pattern.recipe}") - - def _is_valid_pattern(self, pattern:FileEventPattern)->None: - """Validation check for 'pattern' variable from main constructor. Is - automatically called during initialisation.""" - check_type( - pattern, - FileEventPattern, - hint="FileEventJupyterNotebookRule.pattern" - ) - - def _is_valid_recipe(self, recipe:JupyterNotebookRecipe)->None: - """Validation check for 'recipe' variable from main constructor. Is - automatically called during initialisation.""" - check_type( - recipe, - JupyterNotebookRecipe, - hint="FileEventJupyterNotebookRule.recipe" - ) diff --git a/rules/file_event_python_rule.py b/rules/file_event_python_rule.py deleted file mode 100644 index 48e4e78..0000000 --- a/rules/file_event_python_rule.py +++ /dev/null @@ -1,39 +0,0 @@ - -""" -This file contains definitions for a MEOW rule connecting the FileEventPattern -and PythonRecipe. - -Author(s): David Marchant -""" - -from meow_base.core.base_rule import BaseRule -from meow_base.core.correctness.validation import check_type -from meow_base.patterns.file_event_pattern import FileEventPattern -from meow_base.recipes.python_recipe import PythonRecipe - -# TODO potentailly remove this and just invoke BaseRule directly, as does not -# add any functionality other than some validation. -class FileEventPythonRule(BaseRule): - pattern_type = "FileEventPattern" - recipe_type = "PythonRecipe" - def __init__(self, name: str, pattern:FileEventPattern, - recipe:PythonRecipe): - super().__init__(name, pattern, recipe) - - def _is_valid_pattern(self, pattern:FileEventPattern)->None: - """Validation check for 'pattern' variable from main constructor. Is - automatically called during initialisation.""" - check_type( - pattern, - FileEventPattern, - hint="FileEventPythonRule.pattern" - ) - - def _is_valid_recipe(self, recipe:PythonRecipe)->None: - """Validation check for 'recipe' variable from main constructor. Is - automatically called during initialisation.""" - check_type( - recipe, - PythonRecipe, - hint="FileEventPythonRule.recipe" - ) diff --git a/tests/shared.py b/tests/shared.py index b48ed37..d53dc4e 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -46,34 +46,36 @@ def backup_before_teardown(backup_source:str, backup_dest:str): copy_tree(backup_source, backup_dest) -# Recipe funcs -BAREBONES_PYTHON_SCRIPT = [ +# Bash scripts +BAREBONES_BASH_SCRIPT = [ "" ] -COMPLETE_PYTHON_SCRIPT = [ - "import os", - "# Setup parameters", - "num = 1000", - "infile = 'somehere"+ os.path.sep +"particular'", - "outfile = 'nowhere"+ os.path.sep +"particular'", - "", - "with open(infile, 'r') as file:", - " s = float(file.read())", - "" - "for i in range(num):", - " s += i", - "", - "div_by = 4", - "result = s / div_by", - "", - "print(result)", - "", - "os.makedirs(os.path.dirname(outfile), exist_ok=True)", - "", - "with open(outfile, 'w') as file:", - " file.write(str(result))", - "", - "print('done')" +COMPLETE_BASH_SCRIPT = [ + '#!/bin/bash', + '', + 'num=1000', + 'infile="data"', + 'outfile="output"', + '', + 'echo "starting"', + '', + 'read var < $infile', + 'for (( i=0; i<$num; i++ ))', + 'do', + ' var=$((var+i))', + 'done', + '', + 'div_by=4', + 'echo $var', + 'result=$((var/div_by))', + '', + 'echo $result', + '', + 'mkdir -p $(dirname $outfile)', + '', + 'echo $result > $outfile', + '', + 'echo "done"' ] # Jupyter notebooks @@ -1225,6 +1227,34 @@ GENERATOR_NOTEBOOK = { } # Python scripts +BAREBONES_PYTHON_SCRIPT = [ + "" +] +COMPLETE_PYTHON_SCRIPT = [ + "import os", + "# Setup parameters", + "num = 1000", + "infile = 'somehere"+ os.path.sep +"particular'", + "outfile = 'nowhere"+ os.path.sep +"particular'", + "", + "with open(infile, 'r') as file:", + " s = float(file.read())", + "" + "for i in range(num):", + " s += i", + "", + "div_by = 4", + "result = s / div_by", + "", + "print(result)", + "", + "os.makedirs(os.path.dirname(outfile), exist_ok=True)", + "", + "with open(outfile, 'w') as file:", + " file.write(str(result))", + "", + "print('done')" +] IDMC_UTILS_MODULE = [ "import matplotlib.pyplot as plt", "from sklearn import mixture", diff --git a/tests/test_base.py b/tests/test_base.py index 4c7c9bf..89deb67 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -8,10 +8,9 @@ from meow_base.core.base_handler import BaseHandler from meow_base.core.base_monitor import BaseMonitor from meow_base.core.base_pattern import BasePattern from meow_base.core.base_recipe import BaseRecipe -from meow_base.core.base_rule import BaseRule from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START from meow_base.patterns.file_event_pattern import FileEventPattern -from shared import valid_pattern_one, valid_recipe_one, setup, teardown +from shared import setup, teardown class BaseRecipeTests(unittest.TestCase): @@ -147,35 +146,6 @@ class BasePatternTests(unittest.TestCase): self.assertEqual(len(values), 0) -class BaseRuleTests(unittest.TestCase): - def setUp(self)->None: - super().setUp() - setup() - - def tearDown(self)->None: - super().tearDown() - teardown() - - # Test that BaseRecipe instantiation - def testBaseRule(self)->None: - with self.assertRaises(TypeError): - BaseRule("name", "", "") - - class NewRule(BaseRule): - pass - with self.assertRaises(NotImplementedError): - NewRule("name", "", "") - - class FullRule(BaseRule): - pattern_type = "pattern" - recipe_type = "recipe" - def _is_valid_recipe(self, recipe:Any)->None: - pass - def _is_valid_pattern(self, pattern:Any)->None: - pass - FullRule("name", valid_pattern_one, valid_recipe_one) - - class BaseMonitorTests(unittest.TestCase): def setUp(self)->None: super().setUp() diff --git a/tests/test_functionality.py b/tests/test_functionality.py index ce3bc1c..d6c8fc9 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -11,7 +11,7 @@ from sys import prefix, base_prefix from time import sleep from typing import Dict -from meow_base.core.base_rule import BaseRule +from meow_base.core.rule import Rule from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \ WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \ @@ -581,7 +581,7 @@ class MeowTests(unittest.TestCase): def testCreateRule(self)->None: rule = create_rule(valid_pattern_one, valid_recipe_one) - self.assertIsInstance(rule, BaseRule) + self.assertIsInstance(rule, Rule) with self.assertRaises(ValueError): rule = create_rule(valid_pattern_one, valid_recipe_two) @@ -607,7 +607,7 @@ class MeowTests(unittest.TestCase): self.assertEqual(len(rules), 2) for k, rule in rules.items(): self.assertIsInstance(k, str) - self.assertIsInstance(rule, BaseRule) + self.assertIsInstance(rule, Rule) self.assertEqual(k, rule.name) # Test that create_rules creates nothing from invalid pattern inputs diff --git a/tests/test_recipes.py b/tests/test_recipes.py index f007768..0f43c39 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,6 +1,8 @@ import jsonschema import os +import stat +import subprocess import unittest from multiprocessing import Pipe @@ -11,22 +13,27 @@ from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, \ EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \ JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \ JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \ - SWEEP_START, JOB_TYPE_PAPERMILL, get_base_file, get_job_file, \ - get_result_file + SWEEP_START, JOB_TYPE_PAPERMILL, JOB_TYPE_BASH, \ + get_base_file, get_job_file, get_result_file +from meow_base.core.rule import Rule from meow_base.functionality.file_io import lines_to_string, make_dir, \ read_yaml, write_file, write_notebook, write_yaml from meow_base.functionality.hashing import get_file_hash from meow_base.functionality.meow import create_job, create_rules, \ create_rule, create_watchdog_event +from meow_base.functionality.parameterisation import parameterize_bash_script from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.bash_recipe import BashRecipe, BashHandler, \ + assemble_bash_job_script from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler, papermill_job_func, get_recipe_from_notebook from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, \ python_job_func -from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule from shared import BAREBONES_PYTHON_SCRIPT, COMPLETE_PYTHON_SCRIPT, \ TEST_JOB_QUEUE, TEST_MONITOR_BASE, TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, \ - APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, setup, teardown + APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, BAREBONES_BASH_SCRIPT, \ + COMPLETE_BASH_SCRIPT, \ + setup, teardown class JupyterNotebookTests(unittest.TestCase): def setUp(self)->None: @@ -157,7 +164,7 @@ class PapermillHandlerTests(unittest.TestCase): rules = create_rules(patterns, recipes) self.assertEqual(len(rules), 1) _, rule = rules.popitem() - self.assertIsInstance(rule, FileEventJupyterNotebookRule) + self.assertIsInstance(rule, Rule) self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) @@ -208,7 +215,7 @@ class PapermillHandlerTests(unittest.TestCase): rules = create_rules(patterns, recipes) self.assertEqual(len(rules), 1) _, rule = rules.popitem() - self.assertIsInstance(rule, FileEventJupyterNotebookRule) + self.assertIsInstance(rule, Rule) self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) @@ -278,7 +285,7 @@ class PapermillHandlerTests(unittest.TestCase): rules = create_rules(patterns, recipes) self.assertEqual(len(rules), 1) _, rule = rules.popitem() - self.assertIsInstance(rule, FileEventJupyterNotebookRule) + self.assertIsInstance(rule, Rule) self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) @@ -576,7 +583,7 @@ class PythonHandlerTests(unittest.TestCase): rules = create_rules(patterns, recipes) self.assertEqual(len(rules), 1) _, rule = rules.popitem() - self.assertIsInstance(rule, FileEventPythonRule) + self.assertIsInstance(rule, Rule) self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) @@ -627,7 +634,7 @@ class PythonHandlerTests(unittest.TestCase): rules = create_rules(patterns, recipes) self.assertEqual(len(rules), 1) _, rule = rules.popitem() - self.assertIsInstance(rule, FileEventPythonRule) + self.assertIsInstance(rule, Rule) self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) @@ -697,7 +704,7 @@ class PythonHandlerTests(unittest.TestCase): rules = create_rules(patterns, recipes) self.assertEqual(len(rules), 1) _, rule = rules.popitem() - self.assertIsInstance(rule, FileEventPythonRule) + self.assertIsInstance(rule, Rule) self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) @@ -830,6 +837,12 @@ class PythonHandlerTests(unittest.TestCase): self.assertTrue(os.path.exists(result_path)) + with open(result_path, "r") as f: + result = f.read() + + self.assertEqual(result, "124937.5") + + # Test jobFunc doesn't execute with no args def testJobFuncBadArgs(self)->None: try: @@ -878,3 +891,425 @@ class PythonHandlerTests(unittest.TestCase): EVENT_RULE: rule }) self.assertTrue(status) + +class BashTests(unittest.TestCase): + def setUp(self)->None: + super().setUp() + setup() + + def tearDown(self)->None: + super().tearDown() + teardown() + + # Test BashRecipe can be created + def testBashRecipeCreationMinimum(self)->None: + BashRecipe("test_recipe", BAREBONES_BASH_SCRIPT) + + # Test BashRecipe cannot be created without name + def testBashRecipeCreationNoName(self)->None: + with self.assertRaises(ValueError): + BashRecipe("", BAREBONES_BASH_SCRIPT) + + # Test BashRecipe cannot be created with invalid name + def testBashRecipeCreationInvalidName(self)->None: + with self.assertRaises(ValueError): + BashRecipe("@test_recipe", BAREBONES_BASH_SCRIPT) + + # Test BashRecipe cannot be created with invalid recipe + def testBashRecipeCreationInvalidRecipe(self)->None: + with self.assertRaises(TypeError): + BashRecipe("test_recipe", BAREBONES_NOTEBOOK) + + # Test BashRecipe name setup correctly + def testBashRecipeSetupName(self)->None: + name = "name" + pr = BashRecipe(name, BAREBONES_BASH_SCRIPT) + self.assertEqual(pr.name, name) + + # Test BashRecipe recipe setup correctly + def testBashRecipeSetupRecipe(self)->None: + pr = BashRecipe("name", BAREBONES_BASH_SCRIPT) + self.assertEqual(pr.recipe, BAREBONES_BASH_SCRIPT) + + # Test BashRecipe parameters setup correctly + def testBashRecipeSetupParameters(self)->None: + parameters = { + "a": 1, + "b": True + } + pr = BashRecipe( + "name", BAREBONES_BASH_SCRIPT, parameters=parameters) + self.assertEqual(pr.parameters, parameters) + + # Test BashRecipe requirements setup correctly + def testBashRecipeSetupRequirements(self)->None: + requirements = { + "a": 1, + "b": True + } + pr = BashRecipe( + "name", BAREBONES_BASH_SCRIPT, requirements=requirements) + self.assertEqual(pr.requirements, requirements) + +class BashHandlerTests(unittest.TestCase): + def setUp(self)->None: + super().setUp() + setup() + + def tearDown(self)->None: + super().tearDown() + teardown() + + # Test BashHandler can be created + def testBashHandlerMinimum(self)->None: + BashHandler(job_queue_dir=TEST_JOB_QUEUE) + + # Test BashHandler naming + def testBashHandlerNaming(self)->None: + test_name = "test_name" + handler = BashHandler(name=test_name) + self.assertEqual(handler.name, test_name) + + handler = BashHandler() + self.assertTrue(handler.name.startswith("handler_")) + + # Test BashHandler will handle given events + def testBashHandlerHandling(self)->None: + from_handler_reader, from_handler_writer = Pipe() + ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE) + ph.to_runner = from_handler_writer + + with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f: + f.write("Data") + + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one") + recipe = BashRecipe( + "recipe_one", COMPLETE_BASH_SCRIPT) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + rules = create_rules(patterns, recipes) + self.assertEqual(len(rules), 1) + _, rule = rules.popitem() + self.assertIsInstance(rule, Rule) + + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + event = { + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), + WATCHDOG_BASE: TEST_MONITOR_BASE, + EVENT_RULE: rule, + WATCHDOG_HASH: get_file_hash( + os.path.join(TEST_MONITOR_BASE, "A"), SHA256 + ) + } + + ph.handle(event) + + if from_handler_reader.poll(3): + job_dir = from_handler_reader.recv() + + self.assertIsInstance(job_dir, str) + self.assertTrue(os.path.exists(job_dir)) + + job = read_yaml(os.path.join(job_dir, META_FILE)) + valid_job(job) + + # Test BashHandler will create enough jobs from single sweep + def testBashHandlerHandlingSingleSweep(self)->None: + from_handler_reader, from_handler_writer = Pipe() + ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE) + ph.to_runner = from_handler_writer + + with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f: + f.write("Data") + + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one", sweep={"s":{ + SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1 + }}) + recipe = BashRecipe( + "recipe_one", COMPLETE_BASH_SCRIPT) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + rules = create_rules(patterns, recipes) + self.assertEqual(len(rules), 1) + _, rule = rules.popitem() + self.assertIsInstance(rule, Rule) + + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + event = { + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), + WATCHDOG_BASE: TEST_MONITOR_BASE, + EVENT_RULE: rule, + WATCHDOG_HASH: get_file_hash( + os.path.join(TEST_MONITOR_BASE, "A"), SHA256 + ) + } + + ph.handle(event) + + jobs = [] + recieving = True + while recieving: + if from_handler_reader.poll(3): + jobs.append(from_handler_reader.recv()) + else: + recieving = False + + values = [0, 1, 2] + self.assertEqual(len(jobs), 3) + for job_dir in jobs: + self.assertIsInstance(job_dir, str) + self.assertTrue(os.path.exists(job_dir)) + + job = read_yaml(os.path.join(job_dir, META_FILE)) + valid_job(job) + + self.assertIn(JOB_PARAMETERS, job) + self.assertIn("s", job[JOB_PARAMETERS]) + if job[JOB_PARAMETERS]["s"] in values: + values.remove(job[JOB_PARAMETERS]["s"]) + self.assertEqual(len(values), 0) + + # Test BashHandler will create enough jobs from multiple sweeps + def testBashHandlerHandlingMultipleSweep(self)->None: + from_handler_reader, from_handler_writer = Pipe() + ph = BashHandler(job_queue_dir=TEST_JOB_QUEUE) + ph.to_runner = from_handler_writer + + with open(os.path.join(TEST_MONITOR_BASE, "A"), "w") as f: + f.write("Data") + + pattern_one = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one", sweep={ + "s1":{ + SWEEP_START: 0, SWEEP_STOP: 2, SWEEP_JUMP:1 + }, + "s2":{ + SWEEP_START: 20, SWEEP_STOP: 80, SWEEP_JUMP:15 + } + }) + recipe = BashRecipe( + "recipe_one", COMPLETE_BASH_SCRIPT) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + rules = create_rules(patterns, recipes) + self.assertEqual(len(rules), 1) + _, rule = rules.popitem() + self.assertIsInstance(rule, Rule) + + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + event = { + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_PATH: os.path.join(TEST_MONITOR_BASE, "A"), + WATCHDOG_BASE: TEST_MONITOR_BASE, + EVENT_RULE: rule, + WATCHDOG_HASH: get_file_hash( + os.path.join(TEST_MONITOR_BASE, "A"), SHA256 + ) + } + + ph.handle(event) + + jobs = [] + recieving = True + while recieving: + if from_handler_reader.poll(3): + jobs.append(from_handler_reader.recv()) + else: + recieving = False + + values = [ + "s1-0/s2-20", "s1-1/s2-20", "s1-2/s2-20", + "s1-0/s2-35", "s1-1/s2-35", "s1-2/s2-35", + "s1-0/s2-50", "s1-1/s2-50", "s1-2/s2-50", + "s1-0/s2-65", "s1-1/s2-65", "s1-2/s2-65", + "s1-0/s2-80", "s1-1/s2-80", "s1-2/s2-80", + ] + self.assertEqual(len(jobs), 15) + for job_dir in jobs: + self.assertIsInstance(job_dir, str) + self.assertTrue(os.path.exists(job_dir)) + + job = read_yaml(os.path.join(job_dir, META_FILE)) + valid_job(job) + + self.assertIn(JOB_PARAMETERS, job) + val1 = None + val2 = None + if "s1" in job[JOB_PARAMETERS]: + val1 = f"s1-{job[JOB_PARAMETERS]['s1']}" + if "s2" in job[JOB_PARAMETERS]: + val2 = f"s2-{job[JOB_PARAMETERS]['s2']}" + val = None + if val1 and val2: + val = f"{val1}/{val2}" + if val and val in values: + values.remove(val) + self.assertEqual(len(values), 0) + + # Test jobFunc performs as expected + def testJobFunc(self)->None: + file_path = os.path.join(TEST_MONITOR_BASE, "test") + result_path = os.path.join(TEST_MONITOR_BASE, "output") + + with open(file_path, "w") as f: + f.write("250") + + file_hash = get_file_hash(file_path, SHA256) + + pattern = FileEventPattern( + "pattern", + file_path, + "recipe_one", + "infile", + parameters={ + "extra":"A line from a test Pattern", + "outfile": result_path + }) + recipe = BashRecipe( + "recipe_one", COMPLETE_BASH_SCRIPT) + + rule = create_rule(pattern, recipe) + + params_dict = { + "extra":"extra", + "infile":file_path, + "outfile": result_path + } + + job_dict = create_job( + JOB_TYPE_BASH, + create_watchdog_event( + file_path, + rule, + TEST_MONITOR_BASE, + file_hash + ), + extras={ + JOB_PARAMETERS:params_dict, + JOB_HASH: file_hash + } + ) + + job_dir = os.path.join(TEST_JOB_QUEUE, job_dict[JOB_ID]) + make_dir(job_dir) + + meta_file = os.path.join(job_dir, META_FILE) + write_yaml(job_dict, meta_file) + + base_script = parameterize_bash_script( + COMPLETE_BASH_SCRIPT, params_dict + ) + base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_BASH)) + write_file(lines_to_string(base_script), base_file) + st = os.stat(base_file) + os.chmod(base_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + job_script = assemble_bash_job_script() + job_file = os.path.join(job_dir, get_job_file(JOB_TYPE_BASH)) + write_file(lines_to_string(job_script), job_file) + st = os.stat(job_file) + os.chmod(job_file, st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + + print(os.listdir(job_dir)) + print(os.getcwd()) + + result = subprocess.call(job_file, cwd=".") + + self.assertEqual(result, 0) + + self.assertTrue(os.path.exists(job_dir)) + meta_path = os.path.join(job_dir, META_FILE) + self.assertTrue(os.path.exists(meta_path)) + + status = read_yaml(meta_path) + self.assertIsInstance(status, Dict) + self.assertIn(JOB_STATUS, status) + self.assertEqual(status[JOB_STATUS], job_dict[JOB_STATUS]) + self.assertNotIn(JOB_ERROR, status) + + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_base_file(JOB_TYPE_BASH)))) + self.assertTrue(os.path.exists( + os.path.join(job_dir, get_job_file(JOB_TYPE_BASH)))) + + self.assertTrue(os.path.exists(result_path)) + + with open(result_path, "r") as f: + result = f.read() + + self.assertEqual(result, "124937\n") + + # Test jobFunc doesn't execute with no args + def testJobFuncBadArgs(self)->None: + try: + Bash_job_func({}) + except Exception: + pass + + self.assertEqual(len(os.listdir(TEST_JOB_QUEUE)), 0) + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 0) + + # Test handling criteria function + def testValidHandleCriteria(self)->None: + ph = BashHandler() + + pattern = FileEventPattern( + "pattern_one", "A", "recipe_one", "file_one") + recipe = BashRecipe( + "recipe_one", BAREBONES_BASH_SCRIPT + ) + + rule = create_rule(pattern, recipe) + + status, _ = ph.valid_handle_criteria({}) + self.assertFalse(status) + + status, _ = ph.valid_handle_criteria("") + self.assertFalse(status) + + status, _ = ph.valid_handle_criteria({ + EVENT_PATH: "path", + EVENT_TYPE: "type", + EVENT_RULE: rule + }) + self.assertFalse(status) + + status, _ = ph.valid_handle_criteria({ + EVENT_PATH: "path", + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_RULE: "rule" + }) + self.assertFalse(status) + + status, s = ph.valid_handle_criteria({ + EVENT_PATH: "path", + EVENT_TYPE: EVENT_TYPE_WATCHDOG, + EVENT_RULE: rule + }) + self.assertTrue(status) diff --git a/tests/test_rule.py b/tests/test_rule.py new file mode 100644 index 0000000..3e2ef63 --- /dev/null +++ b/tests/test_rule.py @@ -0,0 +1,92 @@ + +import unittest + +from meow_base.core.rule import Rule +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe +from shared import BAREBONES_NOTEBOOK, setup, teardown + +class CorrectnessTests(unittest.TestCase): + def setUp(self)->None: + super().setUp() + setup() + + def tearDown(self)->None: + super().tearDown() + teardown() + + # Test Rule created from valid pattern and recipe + def testRuleCreationMinimum(self)->None: + fep = FileEventPattern("name", "path", "recipe", "file") + jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) + + Rule(fep, jnr) + + # Test Rule not created with empty name + def testRuleCreationNoName(self)->None: + fep = FileEventPattern("name", "path", "recipe", "file") + jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) + + r = Rule(fep, jnr) + + self.assertIsInstance(r.name, str) + self.assertTrue(len(r.name) > 1) + + # Test Rule not created with invalid name + def testRuleCreationInvalidName(self)->None: + fep = FileEventPattern("name", "path", "recipe", "file") + jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) + + with self.assertRaises(TypeError): + Rule(fep, jnr, name=1) + + # Test Rule not created with invalid pattern + def testRuleCreationInvalidPattern(self)->None: + jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) + + with self.assertRaises(TypeError): + Rule("pattern", jnr) + + # Test Rule not created with invalid recipe + def testRuleCreationInvalidRecipe(self)->None: + fep = FileEventPattern("name", "path", "recipe", "file") + + with self.assertRaises(TypeError): + Rule(fep, "recipe") + + # Test Rule not created with mismatched recipe + def testRuleCreationMissmatchedRecipe(self)->None: + fep = FileEventPattern("name", "path", "recipe", "file") + jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK) + + with self.assertRaises(ValueError): + Rule(fep, jnr) + + # Test Rule created with valid name + def testRuleSetupName(self)->None: + name = "name" + fep = FileEventPattern("name", "path", "recipe", "file") + jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) + + fejnr = Rule(fep, jnr, name=name) + + self.assertEqual(fejnr.name, name) + + # Test Rule not created with valid pattern + def testRuleSetupPattern(self)->None: + fep = FileEventPattern("name", "path", "recipe", "file") + jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) + + fejnr = Rule(fep, jnr) + + self.assertEqual(fejnr.pattern, fep) + + # Test Rule not created with valid recipe + def testRuleSetupRecipe(self)->None: + fep = FileEventPattern("name", "path", "recipe", "file") + jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) + + fejnr = Rule(fep, jnr) + + self.assertEqual(fejnr.recipe, jnr) + diff --git a/tests/test_rules.py b/tests/test_rules.py deleted file mode 100644 index d1b9b47..0000000 --- a/tests/test_rules.py +++ /dev/null @@ -1,91 +0,0 @@ - -import unittest - -from meow_base.patterns.file_event_pattern import FileEventPattern -from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe -from meow_base.rules.file_event_jupyter_notebook_rule import \ - FileEventJupyterNotebookRule -from shared import BAREBONES_NOTEBOOK, setup, teardown - -class CorrectnessTests(unittest.TestCase): - def setUp(self)->None: - super().setUp() - setup() - - def tearDown(self)->None: - super().tearDown() - teardown() - - # Test FileEventJupyterNotebookRule created from valid pattern and recipe - def testFileEventJupyterNotebookRuleCreationMinimum(self)->None: - fep = FileEventPattern("name", "path", "recipe", "file") - jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) - - FileEventJupyterNotebookRule("name", fep, jnr) - - # Test FileEventJupyterNotebookRule not created with empty name - def testFileEventJupyterNotebookRuleCreationNoName(self)->None: - fep = FileEventPattern("name", "path", "recipe", "file") - jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) - - with self.assertRaises(ValueError): - FileEventJupyterNotebookRule("", fep, jnr) - - # Test FileEventJupyterNotebookRule not created with invalid name - def testFileEventJupyterNotebookRuleCreationInvalidName(self)->None: - fep = FileEventPattern("name", "path", "recipe", "file") - jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) - - with self.assertRaises(TypeError): - FileEventJupyterNotebookRule(1, fep, jnr) - - # Test FileEventJupyterNotebookRule not created with invalid pattern - def testFileEventJupyterNotebookRuleCreationInvalidPattern(self)->None: - jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) - - with self.assertRaises(TypeError): - FileEventJupyterNotebookRule("name", "pattern", jnr) - - # Test FileEventJupyterNotebookRule not created with invalid recipe - def testFileEventJupyterNotebookRuleCreationInvalidRecipe(self)->None: - fep = FileEventPattern("name", "path", "recipe", "file") - - with self.assertRaises(TypeError): - FileEventJupyterNotebookRule("name", fep, "recipe") - - # Test FileEventJupyterNotebookRule not created with mismatched recipe - def testFileEventJupyterNotebookRuleCreationMissmatchedRecipe(self)->None: - fep = FileEventPattern("name", "path", "recipe", "file") - jnr = JupyterNotebookRecipe("test_recipe", BAREBONES_NOTEBOOK) - - with self.assertRaises(ValueError): - FileEventJupyterNotebookRule("name", fep, jnr) - - # Test FileEventJupyterNotebookRule created with valid name - def testFileEventJupyterNotebookRuleSetupName(self)->None: - name = "name" - fep = FileEventPattern("name", "path", "recipe", "file") - jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) - - fejnr = FileEventJupyterNotebookRule(name, fep, jnr) - - self.assertEqual(fejnr.name, name) - - # Test FileEventJupyterNotebookRule not created with valid pattern - def testFileEventJupyterNotebookRuleSetupPattern(self)->None: - fep = FileEventPattern("name", "path", "recipe", "file") - jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) - - fejnr = FileEventJupyterNotebookRule("name", fep, jnr) - - self.assertEqual(fejnr.pattern, fep) - - # Test FileEventJupyterNotebookRule not created with valid recipe - def testFileEventJupyterNotebookRuleSetupRecipe(self)->None: - fep = FileEventPattern("name", "path", "recipe", "file") - jnr = JupyterNotebookRecipe("recipe", BAREBONES_NOTEBOOK) - - fejnr = FileEventJupyterNotebookRule("name", fep, jnr) - - self.assertEqual(fejnr.recipe, jnr) -