diff --git a/benchmarking/mrme.py b/benchmarking/mrme.py index 7a79e8b..513af7b 100644 --- a/benchmarking/mrme.py +++ b/benchmarking/mrme.py @@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook from shared import run_test, MRME -def multiple_rules_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): +def multiple_rules_multiple_events(job_count:int, REPEATS:int, job_counter:int, + requested_jobs:int, runtime_start:float): patterns = {} for i in range(job_count): pattern = FileEventPattern( diff --git a/benchmarking/mrse.py b/benchmarking/mrse.py index def47cd..d6f34fa 100644 --- a/benchmarking/mrse.py +++ b/benchmarking/mrse.py @@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook from shared import run_test, MRSE -def multiple_rules_single_event(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): +def multiple_rules_single_event(job_count:int, REPEATS:int, job_counter:int, + requested_jobs:int, runtime_start:float): patterns = {} for i in range(job_count): pattern = FileEventPattern( diff --git a/benchmarking/run_all.py b/benchmarking/run_all.py index e4a8f81..3b343f6 100644 --- a/benchmarking/run_all.py +++ b/benchmarking/run_all.py @@ -5,14 +5,18 @@ import sys import time import os -from shared import JOBS_COUNTS, REPEATS, TESTS, MRME, MRSE, SRME, SRSEP, SRSES, RESULTS_DIR, BASE, GRAPH_FILENAME +from typing import List + +from shared import JOBS_COUNTS, REPEATS, TESTS, MRME, MRSE, SRME, SRSEP, \ + SRSES, RESULTS_DIR, BASE from mrme import multiple_rules_multiple_events from mrse import multiple_rules_single_event from srme import single_rule_multiple_events from srsep import single_rule_single_event_parallel from srsps import single_rule_single_event_sequential -from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR +from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \ + DEFAULT_JOB_QUEUE_DIR from meow_base.functionality.file_io import rmtree LINE_KEYS = { @@ -59,7 +63,7 @@ def run_tests(): print(f"All tests completed in: {str(time.time()-runtime_start)}") -def get_meow_graph(results_dir): +def get_meow_graph(results_dir:str): lines = [] for run_type in os.listdir(results_dir): @@ -68,24 +72,25 @@ def get_meow_graph(results_dir): # lines.append((f'scheduling {run_type}', [], 'solid')) lines.append((run_type, [], 'solid')) - run_type_path = os.path.join(results_dir, run_type) + run_path = os.path.join(results_dir, run_type) - for job_count in os.listdir(run_type_path): - results_path = os.path.join(run_type_path, job_count, 'results.txt') + for job_count in os.listdir(run_path): + results_path = os.path.join(run_path, job_count, "results.txt") with open(results_path, 'r') as f_in: data = f_in.readlines() scheduling_duration = 0 for line in data: if "Average schedule time: " in line: - scheduling_duration = float(line.replace("Average schedule time: ", '')) + scheduling_duration = float(line.replace( + "Average schedule time: ", '')) lines[-1][1].append((job_count, scheduling_duration)) lines[-1][1].sort(key=lambda y: float(y[0])) return lines -def make_plot(lines, graph_path, title, logged): +def make_plot(lines:List, graph_path:str, title:str, logged:bool): w = 10 h = 4 linecount = 0 @@ -129,14 +134,19 @@ def make_plot(lines, graph_path, title, logged): def make_both_plots(lines, path, title, log=True): make_plot(lines, path, title, False) if log: - logged_path = path[:path.index(".pdf")] + "_logged" + path[path.index(".pdf"):] + logged_path = path[:path.index(".pdf")] + "_logged" \ + + path[path.index(".pdf"):] make_plot(lines, logged_path, title, True) def make_graphs(): lines = get_meow_graph(RESULTS_DIR) - make_both_plots(lines, "result.pdf", "MiG scheduling overheads on the Threadripper") + make_both_plots( + lines, + "result.pdf", + "MiG scheduling overheads on the Threadripper" + ) average_lines = [] all_delta_lines = [] @@ -146,20 +156,36 @@ def make_graphs(): averages = [(i, v/float(i)) for i, v in line_values] average_lines.append((line_signature, averages, lines_style)) - if line_signature not in ["total single_Pattern_single_file_sequential", "scheduling single_Pattern_single_file_sequential_jobs", "SPSFS"]: + if line_signature not in [ + "total single_Pattern_single_file_sequential", + "scheduling single_Pattern_single_file_sequential_jobs", + "SPSFS"]: deltas = [] for i in range(len(line_values)-1): - deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) ) + deltas.append((line_values[i+1][0], + (averages[i+1][1]-averages[i][1]) \ + / (float(averages[i+1][0])-float(averages[i][0])))) no_spsfs_delta_lines.append((line_signature, deltas, lines_style)) deltas = [] for i in range(len(line_values)-1): - deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) ) + deltas.append((line_values[i+1][0], + (averages[i+1][1]-averages[i][1]) \ + / (float(averages[i+1][0])-float(averages[i][0])))) all_delta_lines.append((line_signature, deltas, lines_style)) - make_both_plots(average_lines, "result_averaged.pdf", "Per-job MiG scheduling overheads on the Threadripper") + make_both_plots( + average_lines, + "result_averaged.pdf", + "Per-job MiG scheduling overheads on the Threadripper" + ) - make_both_plots(all_delta_lines, "result_deltas.pdf", "Difference in per-job MiG scheduling overheads on the Threadripper", log=False) + make_both_plots( + all_delta_lines, + "result_deltas.pdf", + "Difference in per-job MiG scheduling overheads on the Threadripper", + log=False + ) if __name__ == '__main__': try: @@ -172,4 +198,4 @@ if __name__ == '__main__': try: sys.exit(1) except SystemExit: - os._exit(1) \ No newline at end of file + os._exit(1) diff --git a/benchmarking/shared.py b/benchmarking/shared.py index 8b633bc..afbe9fe 100644 --- a/benchmarking/shared.py +++ b/benchmarking/shared.py @@ -6,9 +6,12 @@ import pathlib import time import yaml -from typing import Any, Dict, Tuple +from typing import Any, Dict, Tuple, List -from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR +from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \ + DEFAULT_JOB_QUEUE_DIR +from meow_base.core.base_pattern import BasePattern +from meow_base.core.base_recipe import BaseRecipe from meow_base.core.runner import MeowRunner from meow_base.patterns.file_event_pattern import WatchdogMonitor from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler @@ -17,9 +20,13 @@ from meow_base.functionality.file_io import rmtree RESULTS_DIR = "results" BASE = "benchmark_base" -GRAPH_FILENAME = "graph.pdf" REPEATS = 10 -JOBS_COUNTS = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, 250, 300, 400, 500] +JOBS_COUNTS = [ + 10, 20, 30, 40, 50, + 60, 70, 80, 90, 100, + 125, 150, 175, 200, + 250, 300, 400, 500 +] SRME = "single_rule_multiple_events" MRSE = "multiple_rules_single_event" @@ -42,10 +49,11 @@ class DummyConductor(LocalPythonConductor): return False, ">:(" -def datetime_to_timestamp(date_time_obj): - return time.mktime(date_time_obj.timetuple()) + float(date_time_obj.microsecond)/1000000 +def datetime_to_timestamp(date_time_obj:datetime): + return time.mktime(date_time_obj.timetuple()) \ + + float(date_time_obj.microsecond)/1000000 -def generate(file_count, file_path, file_type='.txt'): +def generate(file_count:int, file_path:str, file_type:str=".txt"): first_filename = '' start = time.time() for i in range(int(file_count)): @@ -56,7 +64,8 @@ def generate(file_count, file_path, file_type='.txt'): f.write('0') return first_filename, time.time() - start -def cleanup(jobs, file_out, base_time, gen_time, execution=False): +def cleanup(jobs:List[str], file_out:str, base_time:float, gen_time:float, + execution:bool=False): if not jobs: return @@ -116,27 +125,30 @@ def cleanup(jobs, file_out, base_time, gen_time, execution=False): return total_time -def mean(l): +def mean(l:List): return sum(l)/len(l) -def collate_results(base_results_dir): - +def collate_results(base_dir:str): scheduling_delays = [] - for run in os.listdir(base_results_dir): + for run in os.listdir(base_dir): if run != 'results.txt': - with open(os.path.join(base_results_dir, run, 'results.txt'), 'r') as f: + with open(os.path.join(base_dir, run, "results.txt"), 'r') as f: d = f.readlines() for l in d: if "Total scheduling delay (seconds): " in l: - scheduling_delays.append(float(l.replace("Total scheduling delay (seconds): ", ''))) + scheduling_delays.append(float(l.replace( + "Total scheduling delay (seconds): ", ''))) - with open(os.path.join(base_results_dir, 'results.txt'), 'w') as f: + with open(os.path.join(base_dir, 'results.txt'), 'w') as f: f.write(f"Average schedule time: {round(mean(scheduling_delays), 3)}\n") f.write(f"Scheduling times: {scheduling_delays}") -def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_counter, requested_jobs, runtime_start, signature='', execution=False, print_logging=False): +def run_test(patterns:Dict[str,BasePattern], recipes:Dict[str,BaseRecipe], + files_count:int, expected_job_count:int, repeats:int, job_counter:int, + requested_jobs:int, runtime_start:float, signature:str="", + execution:bool=False, print_logging:bool=False): if not os.path.exists(RESULTS_DIR): os.mkdir(RESULTS_DIR) @@ -172,24 +184,12 @@ def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_co print=runner_debug_stream, logging=3 ) - -# meow.WorkflowRunner( -# VGRID, -# num_workers, -# patterns=patterns, -# recipes=recipes, -# daemon=True, -# start_workers=False, -# retro_active_jobs=False, -# print_logging=print_logging, -# file_logging=False, -# wait_time=1 -# ) runner.start() # Generate triggering files - first_filename, generation_duration = generate(files_count, file_base +"/file_") + first_filename, generation_duration = \ + generate(files_count, file_base +"/file_") idle_loops = 0 total_loops = 0 @@ -216,10 +216,27 @@ def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_co else: jobs = os.listdir(DEFAULT_JOB_QUEUE_DIR) - results_path = os.path.join(RESULTS_DIR, signature, str(expected_job_count), str(run), 'results.txt') + results_path = os.path.join( + RESULTS_DIR, + signature, + str(expected_job_count), + str(run), + "results.txt" + ) - cleanup(jobs, results_path, first_filename, generation_duration, execution=execution) + cleanup( + jobs, + results_path, + first_filename, + generation_duration, + execution=execution + ) - print(f"Completed scheduling run {str(run + 1)} of {str(len(jobs))}/{str(expected_job_count)} jobs for '{signature}' {job_counter + expected_job_count*(run+1)}/{requested_jobs} ({str(round(time.time()-runtime_start, 3))}s)") + print(f"Completed scheduling run {str(run + 1)} of {str(len(jobs))}" + f"/{str(expected_job_count)} jobs for '{signature}' " + f"{job_counter + expected_job_count*(run+1)}/{requested_jobs} " + f"({str(round(time.time()-runtime_start, 3))}s)") - collate_results(os.path.join(RESULTS_DIR, signature, str(expected_job_count))) + collate_results( + os.path.join(RESULTS_DIR, signature, str(expected_job_count)) + ) diff --git a/benchmarking/srme.py b/benchmarking/srme.py index b7a0f8e..2309260 100644 --- a/benchmarking/srme.py +++ b/benchmarking/srme.py @@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook from shared import run_test, SRME -def single_rule_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): +def single_rule_multiple_events(job_count:int, REPEATS:int, job_counter:int, + requested_jobs:int, runtime_start:float): patterns = {} pattern = FileEventPattern( f"pattern_one", diff --git a/benchmarking/srsep.py b/benchmarking/srsep.py index d42ae87..77a17a9 100644 --- a/benchmarking/srsep.py +++ b/benchmarking/srsep.py @@ -5,7 +5,8 @@ from meow_base.recipes import get_recipe_from_notebook from meow_base.functionality.meow import create_parameter_sweep from shared import run_test, SRSEP -def single_rule_single_event_parallel(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): +def single_rule_single_event_parallel(job_count:int, REPEATS:int, + job_counter:int, requested_jobs:int, runtime_start:float): patterns = {} pattern = FileEventPattern( f"pattern_one", diff --git a/benchmarking/srsps.py b/benchmarking/srsps.py index 7a18c7e..5ff4e72 100644 --- a/benchmarking/srsps.py +++ b/benchmarking/srsps.py @@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook from shared import run_test, SRSES -def single_rule_single_event_sequential(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): +def single_rule_single_event_sequential(job_count:int, REPEATS:int, + job_counter:int, requested_jobs:int, runtime_start:float): patterns = {} pattern = FileEventPattern( f"pattern_one", diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index 6b87894..c470703 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -13,12 +13,14 @@ from typing import Any, Tuple, Dict from meow_base.core.base_conductor import BaseConductor from meow_base.core.correctness.meow import valid_job -from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ - STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \ - STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \ - JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR +from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, \ + JOB_STATUS, STATUS_RUNNING, JOB_START_TIME, META_FILE, \ + BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \ + JOB_ERROR, JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, \ + DEFAULT_JOB_OUTPUT_DIR from meow_base.core.correctness.validation import valid_dir_path -from meow_base.functionality.file_io import make_dir, read_yaml, write_file, write_yaml +from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \ + write_yaml class LocalPythonConductor(BaseConductor): def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, diff --git a/core/base_handler.py b/core/base_handler.py index b6654cf..b5e9db1 100644 --- a/core/base_handler.py +++ b/core/base_handler.py @@ -8,7 +8,7 @@ Author(s): David Marchant from typing import Any, Tuple, Dict -from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS +from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg from meow_base.core.correctness.validation import check_implementation diff --git a/core/base_monitor.py b/core/base_monitor.py index 8f98035..4743413 100644 --- a/core/base_monitor.py +++ b/core/base_monitor.py @@ -12,7 +12,7 @@ from typing import Union, Dict from meow_base.core.base_pattern import BasePattern from meow_base.core.base_recipe import BaseRecipe from meow_base.core.base_rule import BaseRule -from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS +from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg from meow_base.core.correctness.validation import check_implementation from meow_base.functionality.meow import create_rules diff --git a/core/base_pattern.py b/core/base_pattern.py index ce909e4..cbd32da 100644 --- a/core/base_pattern.py +++ b/core/base_pattern.py @@ -10,8 +10,8 @@ import itertools from typing import Any, Union, Tuple, Dict, List -from meow_base.core.correctness.vars import get_drt_imp_msg, \ - VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP +from meow_base.core.correctness.vars import VALID_PATTERN_NAME_CHARS, \ + SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg from meow_base.core.correctness.validation import valid_string, check_type, \ check_implementation, valid_dict diff --git a/core/base_recipe.py b/core/base_recipe.py index 7d50eeb..fc07657 100644 --- a/core/base_recipe.py +++ b/core/base_recipe.py @@ -8,8 +8,10 @@ Author(s): David Marchant from typing import Any, Dict -from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS -from meow_base.core.correctness.validation import valid_string, check_implementation +from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ + get_drt_imp_msg +from meow_base.core.correctness.validation import check_implementation, \ + valid_string class BaseRecipe: diff --git a/core/base_rule.py b/core/base_rule.py index a87474b..7b44227 100644 --- a/core/base_rule.py +++ b/core/base_rule.py @@ -13,7 +13,8 @@ if "BasePattern" not in modules: from meow_base.core.base_pattern import BasePattern if "BaseRecipe" not in modules: from meow_base.core.base_recipe import BaseRecipe -from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS +from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \ + get_drt_imp_msg from meow_base.core.correctness.validation import valid_string, check_type, \ check_implementation diff --git a/core/correctness/meow.py b/core/correctness/meow.py index ba5b505..28efe9d 100644 --- a/core/correctness/meow.py +++ b/core/correctness/meow.py @@ -4,9 +4,9 @@ from typing import Any, Dict, Type from meow_base.core.base_rule import BaseRule from meow_base.core.correctness.validation import check_type -from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \ - JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \ - JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH +from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \ + JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \ + JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH # Required keys in event dict EVENT_KEYS = { diff --git a/core/runner.py b/core/runner.py index 271a2ea..648e673 100644 --- a/core/runner.py +++ b/core/runner.py @@ -17,10 +17,11 @@ from typing import Any, Union, Dict, List from meow_base.core.base_conductor import BaseConductor from meow_base.core.base_handler import BaseHandler from meow_base.core.base_monitor import BaseMonitor -from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \ - VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR, \ - EVENT_PATH -from meow_base.core.correctness.validation import check_type, valid_list, valid_dir_path +from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, \ + EVENT_TYPE, VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, \ + DEFAULT_JOB_QUEUE_DIR, EVENT_PATH +from meow_base.core.correctness.validation import check_type, valid_list, \ + valid_dir_path from meow_base.functionality.debug import setup_debugging, print_debug from meow_base.functionality.file_io import make_dir, read_yaml from meow_base.functionality.process_io import wait diff --git a/functionality/hashing.py b/functionality/hashing.py index d9855a5..eaee308 100644 --- a/functionality/hashing.py +++ b/functionality/hashing.py @@ -7,7 +7,8 @@ Author(s): David Marchant from hashlib import sha256 from meow_base.core.correctness.vars import HASH_BUFFER_SIZE, SHA256 -from meow_base.core.correctness.validation import check_type, valid_existing_file_path +from meow_base.core.correctness.validation import check_type, \ + valid_existing_file_path def _get_file_sha256(file_path): sha256_hash = sha256() diff --git a/functionality/meow.py b/functionality/meow.py index 96d8efe..21f69de 100644 --- a/functionality/meow.py +++ b/functionality/meow.py @@ -11,12 +11,13 @@ from typing import Any, Dict, Union, List from meow_base.core.base_pattern import BasePattern from meow_base.core.base_recipe import BaseRecipe from meow_base.core.base_rule import BaseRule -from meow_base.core.correctness.validation import check_type, valid_dict, valid_list -from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ - EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \ - JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \ - STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, SWEEP_START, \ - SWEEP_STOP +from meow_base.core.correctness.validation import check_type, valid_dict, \ + valid_list +from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \ + EVENT_TYPE, EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \ + JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \ + JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \ + SWEEP_START, SWEEP_STOP from meow_base.functionality.naming import generate_job_id, generate_rule_id # mig trigger keyword replacements diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index 6c82cda..21f0899 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -23,7 +23,7 @@ from meow_base.core.base_monitor import BaseMonitor from meow_base.core.base_pattern import BasePattern from meow_base.core.base_rule import BaseRule from meow_base.core.correctness.validation import check_type, valid_string, \ - valid_dict, valid_list, valid_path, valid_dir_path + valid_dict, valid_list, valid_dir_path from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \ FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \ @@ -580,29 +580,6 @@ class WatchdogEventHandler(PatternMatchingEventHandler): self.monitor.match(event) -# recent_timestamp = self._recent_jobs[event.src_path] -# difference = event.time_stamp - recent_timestamp -# -# # Discard the event if we already have a recent event at this -# # same path. Update the most recent time, so we can hopefully -# # wait till events have stopped happening -# if difference <= self._settletime: -# self._recent_jobs[event.src_path] = \ -# max(recent_timestamp, event.time_stamp) -# self._recent_jobs_lock.release() -# return -# else: -# self._recent_jobs[event.src_path] = event.time_stamp -# else: -# self._recent_jobs[event.src_path] = event.time_stamp -# except Exception as ex: -# self._recent_jobs_lock.release() -# raise Exception(ex) -# self._recent_jobs_lock.release() -# -# # If we did not have a recent event, then send it on to the monitor -# self.monitor.match(event) - def handle_event(self, event): """Handler function, called by all specific event functions. Will attach a timestamp to the event immediately, and attempt to start a diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index ad0afde..84b7a22 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -16,15 +16,15 @@ from meow_base.core.base_handler import BaseHandler from meow_base.core.correctness.meow import valid_event from meow_base.core.correctness.validation import check_type, valid_string, \ valid_dict, valid_path, valid_dir_path, valid_existing_file_path -from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ - DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \ - EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \ - JOB_ID, WATCHDOG_BASE, META_FILE, \ - PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \ - EVENT_RULE, get_base_file +from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \ + PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, \ + DEFAULT_JOB_QUEUE_DIR, EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, \ + JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, PARAMS_FILE, \ + JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, EVENT_RULE, \ + get_base_file from meow_base.functionality.debug import setup_debugging, print_debug -from meow_base.functionality.file_io import make_dir, read_notebook, write_notebook, \ - write_yaml +from meow_base.functionality.file_io import make_dir, read_notebook, \ + write_notebook, write_yaml from meow_base.functionality.meow import create_job, replace_keywords diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index 7a9ac86..07bf719 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -15,15 +15,15 @@ from meow_base.core.base_handler import BaseHandler from meow_base.core.correctness.meow import valid_event from meow_base.core.correctness.validation import check_script, valid_string, \ valid_dict, valid_dir_path -from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ - DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \ - EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \ - JOB_ID, WATCHDOG_BASE, META_FILE, \ +from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \ + PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, \ + DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, \ + WATCHDOG_HASH, JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, \ PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \ get_base_file from meow_base.functionality.debug import setup_debugging, print_debug -from meow_base.functionality.file_io import make_dir, read_file_lines, write_file, \ - write_yaml, lines_to_string +from meow_base.functionality.file_io import make_dir, read_file_lines, \ + write_file, write_yaml, lines_to_string from meow_base.functionality.meow import create_job, replace_keywords diff --git a/tests/shared.py b/tests/shared.py index 53d50a6..b48ed37 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -7,7 +7,8 @@ import os from distutils.dir_util import copy_tree -from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR +from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \ + DEFAULT_JOB_QUEUE_DIR from meow_base.functionality.file_io import make_dir, rmtree from meow_base.patterns.file_event_pattern import FileEventPattern from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe diff --git a/tests/test_base.py b/tests/test_base.py index cff0f2d..4c7c9bf 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -11,7 +11,7 @@ from meow_base.core.base_recipe import BaseRecipe from meow_base.core.base_rule import BaseRule from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START from meow_base.patterns.file_event_pattern import FileEventPattern -from shared import setup, teardown, valid_pattern_one, valid_recipe_one +from shared import valid_pattern_one, valid_recipe_one, setup, teardown class BaseRecipeTests(unittest.TestCase): diff --git a/tests/test_conductors.py b/tests/test_conductors.py index 10c4d43..6e53354 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -5,24 +5,25 @@ import unittest from datetime import datetime from typing import Dict -from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \ - JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, JOB_EVENT, \ - META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, JOB_PATTERN, \ - STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \ - JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ +from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, \ + JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, \ + JOB_EVENT, META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \ + JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \ + JOB_CREATE_TIME, JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file from meow_base.conductors import LocalPythonConductor from meow_base.functionality.file_io import read_file, read_yaml, write_file, \ write_notebook, write_yaml, lines_to_string, make_dir from meow_base.functionality.hashing import get_file_hash -from meow_base.functionality.meow import create_watchdog_event, create_job, create_rule +from meow_base.functionality.meow import create_watchdog_event, create_job, \ + create_rule from meow_base.patterns.file_event_pattern import FileEventPattern from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ papermill_job_func from meow_base.recipes.python_recipe import PythonRecipe, python_job_func -from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \ - TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, \ - BAREBONES_PYTHON_SCRIPT, BAREBONES_NOTEBOOK +from shared import TEST_MONITOR_BASE, APPENDING_NOTEBOOK, TEST_JOB_OUTPUT, \ + TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, BAREBONES_PYTHON_SCRIPT, \ + BAREBONES_NOTEBOOK, setup, teardown def failing_func(): raise Exception("bad function") @@ -129,7 +130,7 @@ class MeowTests(unittest.TestCase): self.assertTrue(os.path.exists(result_path)) - # Test LocalPythonConductor executes valid papermill jobs + # Test LocalPythonConductor executes valid papermill jobs def testLocalPythonConductorValidPapermillJob(self)->None: lpc = LocalPythonConductor( job_queue_dir=TEST_JOB_QUEUE, diff --git a/tests/test_functionality.py b/tests/test_functionality.py index f709eec..ce3bc1c 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -19,29 +19,27 @@ from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL from meow_base.functionality.debug import setup_debugging -from meow_base.functionality.file_io import lines_to_string, make_dir, read_file, \ - read_file_lines, read_notebook, read_yaml, rmtree, write_file, \ +from meow_base.functionality.file_io import lines_to_string, make_dir, \ + read_file, read_file_lines, read_notebook, read_yaml, rmtree, write_file, \ write_notebook, write_yaml from meow_base.functionality.hashing import get_file_hash -from meow_base.functionality.meow import create_event, create_job, create_rule, \ - create_rules, create_watchdog_event, replace_keywords, \ - create_parameter_sweep, \ - KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \ - KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \ - KEYWORD_REL_PATH +from meow_base.functionality.meow import KEYWORD_BASE, KEYWORD_DIR, \ + KEYWORD_EXTENSION, KEYWORD_FILENAME, KEYWORD_JOB, KEYWORD_PATH, \ + KEYWORD_PREFIX, KEYWORD_REL_DIR, KEYWORD_REL_PATH, \ + create_event, create_job, create_rule, create_rules, \ + create_watchdog_event, replace_keywords, create_parameter_sweep from meow_base.functionality.naming import _generate_id -from meow_base.functionality.parameterisation import parameterize_jupyter_notebook, \ - parameterize_python_script +from meow_base.functionality.parameterisation import \ + parameterize_jupyter_notebook, parameterize_python_script from meow_base.functionality.process_io import wait -from meow_base.functionality.requirements import create_python_requirements, \ - check_requirements, \ - REQUIREMENT_PYTHON, REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, \ - REQ_PYTHON_VERSION +from meow_base.functionality.requirements import REQUIREMENT_PYTHON, \ + REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, REQ_PYTHON_VERSION, \ + create_python_requirements, check_requirements from meow_base.patterns.file_event_pattern import FileEventPattern from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe -from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \ - valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \ - COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT +from shared import TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, \ + COMPLETE_PYTHON_SCRIPT, valid_recipe_two, valid_recipe_one, \ + valid_pattern_one, valid_pattern_two, setup, teardown class DebugTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 2a44032..6359d1c 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -9,10 +9,10 @@ from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \ EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \ SWEEP_JUMP, SWEEP_STOP from meow_base.functionality.file_io import make_dir -from meow_base.patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \ - _DEFAULT_MASK +from meow_base.patterns.file_event_pattern import FileEventPattern, \ + WatchdogMonitor, _DEFAULT_MASK from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe -from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE +from shared import BAREBONES_NOTEBOOK, TEST_MONITOR_BASE, setup, teardown def patterns_equal(tester, pattern_one, pattern_two): diff --git a/tests/test_recipes.py b/tests/test_recipes.py index fdda6bc..044ec60 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -7,26 +7,26 @@ from multiprocessing import Pipe from typing import Dict from meow_base.core.correctness.meow import valid_job -from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ - EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \ - JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \ - META_FILE, JOB_ERROR, \ - PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \ - get_base_file, get_job_file, get_result_file -from meow_base.functionality.file_io import lines_to_string, make_dir, read_yaml, \ - write_file, write_notebook, write_yaml +from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, \ + EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \ + JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \ + JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \ + SWEEP_START, JOB_TYPE_PAPERMILL, get_base_file, get_job_file, \ + get_result_file +from meow_base.functionality.file_io import lines_to_string, make_dir, \ + read_yaml, write_file, write_notebook, write_yaml from meow_base.functionality.hashing import get_file_hash -from meow_base.functionality.meow import create_job, create_rules, create_rule, \ - create_watchdog_event +from meow_base.functionality.meow import create_job, create_rules, \ + create_rule, create_watchdog_event from meow_base.patterns.file_event_pattern import FileEventPattern from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler, papermill_job_func, get_recipe_from_notebook -from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func +from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, \ + python_job_func from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule -from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \ - COMPLETE_PYTHON_SCRIPT, TEST_JOB_QUEUE, TEST_MONITOR_BASE, \ - TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK - +from shared import BAREBONES_PYTHON_SCRIPT, COMPLETE_PYTHON_SCRIPT, \ + TEST_JOB_QUEUE, TEST_MONITOR_BASE, TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, \ + APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, setup, teardown class JupyterNotebookTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_rules.py b/tests/test_rules.py index e3d48ba..d1b9b47 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -3,8 +3,9 @@ import unittest from meow_base.patterns.file_event_pattern import FileEventPattern from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe -from meow_base.rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule -from shared import setup, teardown, BAREBONES_NOTEBOOK +from meow_base.rules.file_event_jupyter_notebook_rule import \ + FileEventJupyterNotebookRule +from shared import BAREBONES_NOTEBOOK, setup, teardown class CorrectnessTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_runner.py b/tests/test_runner.py index 8463f78..b0d2ad6 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -12,23 +12,23 @@ from meow_base.core.base_conductor import BaseConductor from meow_base.core.base_handler import BaseHandler from meow_base.core.base_monitor import BaseMonitor from meow_base.conductors import LocalPythonConductor -from meow_base.core.correctness.vars import get_result_file, \ - JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME +from meow_base.core.correctness.vars import JOB_TYPE_PAPERMILL, JOB_ERROR, \ + META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME, get_result_file from meow_base.core.runner import MeowRunner -from meow_base.functionality.file_io import make_dir, read_file, read_notebook, \ - read_yaml, write_file, lines_to_string +from meow_base.functionality.file_io import make_dir, read_file, \ + read_notebook, read_yaml, write_file, lines_to_string from meow_base.functionality.meow import create_parameter_sweep from meow_base.functionality.requirements import create_python_requirements -from meow_base.patterns.file_event_pattern import WatchdogMonitor, FileEventPattern +from meow_base.patterns.file_event_pattern import WatchdogMonitor, \ + FileEventPattern from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler, \ JupyterNotebookRecipe from meow_base.recipes.python_recipe import PythonHandler, PythonRecipe -from shared import setup, teardown, backup_before_teardown, \ - TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, MAKER_RECIPE, \ - APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, FILTER_RECIPE, \ - POROSITY_CHECK_NOTEBOOK, SEGMENT_FOAM_NOTEBOOK, GENERATOR_NOTEBOOK, \ - FOAM_PORE_ANALYSIS_NOTEBOOK, IDMC_UTILS_MODULE, TEST_DATA, GENERATE_SCRIPT - +from shared import TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, \ + MAKER_RECIPE, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, \ + FILTER_RECIPE, POROSITY_CHECK_NOTEBOOK, SEGMENT_FOAM_NOTEBOOK, \ + GENERATOR_NOTEBOOK, FOAM_PORE_ANALYSIS_NOTEBOOK, IDMC_UTILS_MODULE, \ + TEST_DATA, GENERATE_SCRIPT, setup, teardown, backup_before_teardown pattern_check = FileEventPattern( "pattern_check", diff --git a/tests/test_validation.py b/tests/test_validation.py index a1a6aef..fee10cc 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -5,18 +5,20 @@ import os from datetime import datetime from typing import Any, Union -from meow_base.core.correctness.meow import valid_event, valid_job, valid_watchdog_event -from meow_base.core.correctness.validation import check_type, check_implementation, \ - valid_string, valid_dict, valid_list, valid_existing_file_path, \ - valid_dir_path, valid_non_existing_path, check_callable -from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \ - EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \ - JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \ - WATCHDOG_HASH +from meow_base.core.correctness.meow import valid_event, valid_job, \ + valid_watchdog_event +from meow_base.core.correctness.validation import check_type, \ + check_implementation, valid_string, valid_dict, valid_list, \ + valid_existing_file_path, valid_dir_path, valid_non_existing_path, \ + check_callable +from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, \ + EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \ + JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \ + WATCHDOG_BASE, WATCHDOG_HASH from meow_base.functionality.file_io import make_dir from meow_base.functionality.meow import create_rule -from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \ - valid_recipe_one +from shared import TEST_MONITOR_BASE, valid_pattern_one, valid_recipe_one, \ + setup, teardown class ValidationTests(unittest.TestCase): def setUp(self)->None: