From 40ed98000b322e851052f23316a990061932affd Mon Sep 17 00:00:00 2001 From: PatchOfScotland Date: Mon, 13 Mar 2023 11:32:45 +0100 Subject: [PATCH] reformatted imports to work better on other machines, plus added benchmarking to project --- .gitignore | 4 + benchmarking/mrme.py | 34 ++++ benchmarking/mrse.py | 34 ++++ benchmarking/run_all.py | 175 +++++++++++++++++ benchmarking/sequential.ipynb | 69 +++++++ benchmarking/shared.py | 225 ++++++++++++++++++++++ benchmarking/srme.py | 33 ++++ benchmarking/srsep.py | 35 ++++ benchmarking/srsps.py | 38 ++++ benchmarking/test.ipynb | 47 +++++ conductors/__init__.py | 2 +- conductors/local_python_conductor.py | 10 +- core/__init__.py | 0 core/base_conductor.py | 4 +- core/base_handler.py | 4 +- core/base_monitor.py | 12 +- core/base_pattern.py | 4 +- core/base_recipe.py | 4 +- core/base_rule.py | 8 +- core/correctness/meow.py | 6 +- core/correctness/validation.py | 2 +- core/runner.py | 49 +++-- functionality/__init__.py | 0 functionality/debug.py | 4 +- functionality/file_io.py | 2 +- functionality/hashing.py | 4 +- functionality/meow.py | 14 +- functionality/naming.py | 2 +- functionality/parameterisation.py | 2 +- functionality/process_io.py | 2 +- functionality/requirements.py | 2 +- patterns/__init__.py | 2 +- patterns/file_event_pattern.py | 18 +- recipes/__init__.py | 6 +- recipes/jupyter_notebook_recipe.py | 43 +++-- recipes/python_recipe.py | 24 +-- rules/__init__.py | 4 +- rules/file_event_jupyter_notebook_rule.py | 8 +- rules/file_event_python_rule.py | 8 +- tests/shared.py | 8 +- tests/test_base.py | 16 +- tests/test_conductors.py | 16 +- tests/test_functionality.py | 26 +-- tests/test_patterns.py | 8 +- tests/test_recipes.py | 18 +- tests/test_rules.py | 6 +- tests/test_runner.py | 28 ++- tests/test_validation.py | 10 +- 48 files changed, 907 insertions(+), 173 deletions(-) create mode 100644 benchmarking/mrme.py create mode 100644 benchmarking/mrse.py create mode 100644 benchmarking/run_all.py create mode 100644 benchmarking/sequential.ipynb create mode 100644 benchmarking/shared.py create mode 100644 benchmarking/srme.py create mode 100644 benchmarking/srsep.py create mode 100644 benchmarking/srsps.py create mode 100644 benchmarking/test.ipynb create mode 100644 core/__init__.py create mode 100644 functionality/__init__.py diff --git a/.gitignore b/.gitignore index 0ff3dfe..9415545 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,10 @@ tests/test_data tests/job_output tests/job_queue tests/Backup* +benchmarking/benchmark_base +benchmarking/job_output +benchmarking/job_queue +benchmarking/result* # hdf5 *.h5 diff --git a/benchmarking/mrme.py b/benchmarking/mrme.py new file mode 100644 index 0000000..04c0a61 --- /dev/null +++ b/benchmarking/mrme.py @@ -0,0 +1,34 @@ + +from meow_base.patterns import FileEventPattern +from meow_base.recipes import getRecipeFromNotebook + +from shared import run_test, MRME + +def multiple_rules_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): + patterns = {} + for i in range(job_count): + pattern = FileEventPattern( + f"pattern_{i}", + f"testing/file_{i}.txt", + "recipe_one", + "input" + ) + patterns[pattern.name] = pattern + + recipe = getRecipeFromNotebook("recipe_one", "test.ipynb") + + recipes = { + recipe.name: recipe + } + + run_test( + patterns, + recipes, + job_count, + job_count, + REPEATS, + job_counter, + requested_jobs, + runtime_start, + signature=MRME + ) diff --git a/benchmarking/mrse.py b/benchmarking/mrse.py new file mode 100644 index 0000000..48efb7c --- /dev/null +++ b/benchmarking/mrse.py @@ -0,0 +1,34 @@ + +from meow_base.patterns import FileEventPattern +from meow_base.recipes import getRecipeFromNotebook + +from shared import run_test, MRSE + +def multiple_rules_single_event(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): + patterns = {} + for i in range(job_count): + pattern = FileEventPattern( + f"pattern_{i}", + f"testing/*", + "recipe_one", + "input" + ) + patterns[pattern.name] = pattern + + recipe = getRecipeFromNotebook("recipe_one", "test.ipynb") + + recipes = { + recipe.name: recipe + } + + run_test( + patterns, + recipes, + 1, + job_count, + REPEATS, + job_counter, + requested_jobs, + runtime_start, + signature=MRSE + ) diff --git a/benchmarking/run_all.py b/benchmarking/run_all.py new file mode 100644 index 0000000..e4a8f81 --- /dev/null +++ b/benchmarking/run_all.py @@ -0,0 +1,175 @@ + +import matplotlib.pyplot as pyplot +import numpy +import sys +import time +import os + +from shared import JOBS_COUNTS, REPEATS, TESTS, MRME, MRSE, SRME, SRSEP, SRSES, RESULTS_DIR, BASE, GRAPH_FILENAME +from mrme import multiple_rules_multiple_events +from mrse import multiple_rules_single_event +from srme import single_rule_multiple_events +from srsep import single_rule_single_event_parallel +from srsps import single_rule_single_event_sequential + +from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR +from meow_base.functionality.file_io import rmtree + +LINE_KEYS = { + SRSES: ('x','#a1467e'), + SRME: ('.','#896cff'), + MRME: ('d','#5983b0'), + MRSE: ('P','#ff6cbe'), + SRSEP: ('*','#3faf46'), +} + + +def run_tests(): + rmtree(RESULTS_DIR) + + requested_jobs=0 + for job_count in JOBS_COUNTS: + requested_jobs += job_count * REPEATS * len(TESTS) + print(f"requested_jobs: {requested_jobs}") + + runtime_start=time.time() + + job_counter=0 + for job_count in JOBS_COUNTS: + for test in TESTS: + if test == MRME: + multiple_rules_multiple_events(job_count, REPEATS, job_counter, requested_jobs, runtime_start) + job_counter += job_count * REPEATS + + elif test == MRSE: + multiple_rules_single_event(job_count, REPEATS, job_counter, requested_jobs, runtime_start) + job_counter += job_count * REPEATS + + elif test == SRME: + single_rule_multiple_events(job_count, REPEATS, job_counter, requested_jobs, runtime_start) + job_counter += job_count * REPEATS + + elif test == SRSEP: + single_rule_single_event_parallel(job_count, REPEATS, job_counter, requested_jobs, runtime_start) + job_counter += job_count * REPEATS + + elif test == SRSES: + single_rule_single_event_sequential(job_count, REPEATS, job_counter, requested_jobs, runtime_start) + job_counter += job_count * REPEATS + + print(f"All tests completed in: {str(time.time()-runtime_start)}") + +def get_meow_graph(results_dir): + lines = [] + + for run_type in os.listdir(results_dir): + #if run_type == 'single_Pattern_single_file_sequential': + # continue + +# lines.append((f'scheduling {run_type}', [], 'solid')) + lines.append((run_type, [], 'solid')) + run_type_path = os.path.join(results_dir, run_type) + + for job_count in os.listdir(run_type_path): + results_path = os.path.join(run_type_path, job_count, 'results.txt') + with open(results_path, 'r') as f_in: + data = f_in.readlines() + + scheduling_duration = 0 + for line in data: + if "Average schedule time: " in line: + scheduling_duration = float(line.replace("Average schedule time: ", '')) + + lines[-1][1].append((job_count, scheduling_duration)) + lines[-1][1].sort(key=lambda y: float(y[0])) + + return lines + +def make_plot(lines, graph_path, title, logged): + w = 10 + h = 4 + linecount = 0 + columns = 1 + + pyplot.figure(figsize=(w, h)) + for l in range(len(lines)): + x_values = numpy.asarray([float(i[0]) for i in lines[l][1]]) + y_values = numpy.asarray([float(i[1]) for i in lines[l][1]]) + + # Remove this check to always display lines + if lines[l][2] == 'solid': + pyplot.plot(x_values, y_values, label=lines[l][0], linestyle=lines[l][2], marker=LINE_KEYS[lines[l][0]][0], color=LINE_KEYS[lines[l][0]][1]) + linecount += 1 + + columns = int(linecount/3) + 1 + + pyplot.xlabel("Number of jobs scheduled") + pyplot.ylabel("Time taken (seconds)") + pyplot.title(title) + + handles, labels = pyplot.gca().get_legend_handles_labels() + # legend_order = [2, 4, 0, 1, 3] + # pyplot.legend([handles[i] for i in legend_order], [labels[i] for i in legend_order]) + + pyplot.legend(ncol=columns, prop={'size': 12}) + if logged: + pyplot.yscale('log') + + x_ticks = [] + for tick in x_values: + label = int(tick) + if tick <= 100 and tick % 20 == 0: + label = f"\n{int(tick)}" + x_ticks.append(label) + + pyplot.xticks(x_values, x_ticks) + + pyplot.savefig(graph_path, format='pdf', bbox_inches='tight') + +def make_both_plots(lines, path, title, log=True): + make_plot(lines, path, title, False) + if log: + logged_path = path[:path.index(".pdf")] + "_logged" + path[path.index(".pdf"):] + make_plot(lines, logged_path, title, True) + + +def make_graphs(): + lines = get_meow_graph(RESULTS_DIR) + + make_both_plots(lines, "result.pdf", "MiG scheduling overheads on the Threadripper") + + average_lines = [] + all_delta_lines = [] + no_spsfs_delta_lines = [] + for line_signature, line_values, lines_style in lines: + if lines_style == 'solid': + averages = [(i, v/float(i)) for i, v in line_values] + average_lines.append((line_signature, averages, lines_style)) + + if line_signature not in ["total single_Pattern_single_file_sequential", "scheduling single_Pattern_single_file_sequential_jobs", "SPSFS"]: + deltas = [] + for i in range(len(line_values)-1): + deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) ) + no_spsfs_delta_lines.append((line_signature, deltas, lines_style)) + deltas = [] + for i in range(len(line_values)-1): + deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) ) + all_delta_lines.append((line_signature, deltas, lines_style)) + + + make_both_plots(average_lines, "result_averaged.pdf", "Per-job MiG scheduling overheads on the Threadripper") + + make_both_plots(all_delta_lines, "result_deltas.pdf", "Difference in per-job MiG scheduling overheads on the Threadripper", log=False) + +if __name__ == '__main__': + try: + run_tests() + make_graphs() + rmtree(DEFAULT_JOB_QUEUE_DIR) + rmtree(DEFAULT_JOB_OUTPUT_DIR) + rmtree(BASE) + except KeyboardInterrupt as ki: + try: + sys.exit(1) + except SystemExit: + os._exit(1) \ No newline at end of file diff --git a/benchmarking/sequential.ipynb b/benchmarking/sequential.ipynb new file mode 100644 index 0000000..7e02a29 --- /dev/null +++ b/benchmarking/sequential.ipynb @@ -0,0 +1,69 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "INPUT_FILE = 'file_0.txt'\n", + "MAX_COUNT = 100" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "with open(INPUT_FILE, 'r') as f:\n", + " data = int(f.read())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"read in: \"+ str(data))\n", + "print(\"writing out: \"+ str(data+1))" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "if data+1 < MAX_COUNT:\n", + " with open(INPUT_FILE.replace(str(data), str(data+1)), 'w') as f:\n", + " f.write(str(data+1))" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/benchmarking/shared.py b/benchmarking/shared.py new file mode 100644 index 0000000..8b633bc --- /dev/null +++ b/benchmarking/shared.py @@ -0,0 +1,225 @@ + +import datetime +import io +import os +import pathlib +import time +import yaml + +from typing import Any, Dict, Tuple + +from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR +from meow_base.core.runner import MeowRunner +from meow_base.patterns.file_event_pattern import WatchdogMonitor +from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler +from meow_base.conductors import LocalPythonConductor +from meow_base.functionality.file_io import rmtree + +RESULTS_DIR = "results" +BASE = "benchmark_base" +GRAPH_FILENAME = "graph.pdf" +REPEATS = 10 +JOBS_COUNTS = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, 250, 300, 400, 500] + +SRME = "single_rule_multiple_events" +MRSE = "multiple_rules_single_event" +SRSEP = "single_rule_single_event_parallel" +MRME = "multiple_rules_multiple_events" +SRSES = "single_rule_single_event_sequential" + +TESTS = [ + SRME, + MRSE, + SRSEP, + MRME, + # This test will take approx 90% of total time + SRSES +] + + +class DummyConductor(LocalPythonConductor): + def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]: + return False, ">:(" + + +def datetime_to_timestamp(date_time_obj): + return time.mktime(date_time_obj.timetuple()) + float(date_time_obj.microsecond)/1000000 + +def generate(file_count, file_path, file_type='.txt'): + first_filename = '' + start = time.time() + for i in range(int(file_count)): + filename = file_path + str(i) + file_type + if not first_filename: + first_filename = filename + with open(filename, 'w') as f: + f.write('0') + return first_filename, time.time() - start + +def cleanup(jobs, file_out, base_time, gen_time, execution=False): + if not jobs: + return + + job_timestamps = [] + for job in jobs: + if execution: + with open(f"{DEFAULT_JOB_OUTPUT_DIR}/{job}/job.yml", 'r') as f_in: + data = yaml.load(f_in, Loader=yaml.Loader) + else: + with open(f"{DEFAULT_JOB_QUEUE_DIR}/{job}/job.yml", 'r') as f_in: + data = yaml.load(f_in, Loader=yaml.Loader) + create_datetime = data['create'] + create_timestamp = datetime_to_timestamp(create_datetime) + job_timestamps.append((create_timestamp, create_datetime)) + + job_timestamps.sort(key=lambda y: int(y[0])) + + first = job_timestamps[0] + last = job_timestamps[-1] + + #dt = datetime.datetime.fromtimestamp(os.path.getctime(base_time), datetime.timezone(datetime.timedelta(hours=0))) + dt = datetime.datetime.fromtimestamp(os.path.getctime(base_time)) + +# if execution: +# queue_times = [] +# execution_times = [] +# for j in jobs: +# mrsl_dict = load(os.path.join(mrsl_dir, j))# +# +# queue_times.append(time.mktime(mrsl_dict['EXECUTING_TIMESTAMP']) - time.mktime(mrsl_dict['QUEUED_TIMESTAMP'])) +# execution_times.append(time.mktime(mrsl_dict['FINISHED_TIMESTAMP']) - time.mktime(mrsl_dict['EXECUTING_TIMESTAMP'])) + pathlib.Path(os.path.dirname(file_out)).mkdir(parents=True, exist_ok=True) + with open(file_out, 'w') as f_out: + f_out.write("Job count: "+ str(len(jobs)) +"\n") + f_out.write("Generation time: "+ str(round(gen_time, 5)) +"\n") + f_out.write("First trigger: "+ str(dt) +"\n") + f_out.write("First scheduling datetime: "+ str(first[1]) +"\n") + f_out.write("Last scheduling datetime: "+ str(last[1]) +"\n") + f_out.write("First scheduling unixtime: "+ str(first[0]) +"\n") + f_out.write("First scheduling unixtime: "+ str(last[0]) +"\n") + f_out.write("Scheduling difference (seconds): "+ str(round(last[0] - first[0], 3)) +"\n") + f_out.write("Initial scheduling delay (seconds): "+ str(round(first[0] - os.path.getctime(base_time), 3)) +"\n") + total_time = round(last[0] - os.path.getctime(base_time), 3) + f_out.write("Total scheduling delay (seconds): "+ str(total_time) +"\n") + +# if execution: +# f_out.write("Average execution time (seconds): "+ str(round(mean(execution_times), 3)) +"\n") +# f_out.write("Max execution time (seconds): "+ str(round(max(execution_times), 3)) +"\n") +# f_out.write("Min execution time (seconds): "+ str(round(min(execution_times), 3)) +"\n") + +# f_out.write("Average queueing delay (seconds): "+ str(round(mean(queue_times), 3)) +"\n") +# f_out.write("Max queueing delay (seconds): "+ str(round(max(queue_times), 3)) +"\n") +# f_out.write("Min queueing delay (seconds): "+ str(round(min(queue_times), 3)) +"\n") + +# queue_times.remove(max(queue_times)) +# f_out.write("Average excluded queueing delay (seconds): "+ str(round(mean(queue_times), 3)) +"\n") + + return total_time + +def mean(l): + return sum(l)/len(l) + +def collate_results(base_results_dir): + + scheduling_delays = [] + + for run in os.listdir(base_results_dir): + if run != 'results.txt': + with open(os.path.join(base_results_dir, run, 'results.txt'), 'r') as f: + d = f.readlines() + + for l in d: + if "Total scheduling delay (seconds): " in l: + scheduling_delays.append(float(l.replace("Total scheduling delay (seconds): ", ''))) + + with open(os.path.join(base_results_dir, 'results.txt'), 'w') as f: + f.write(f"Average schedule time: {round(mean(scheduling_delays), 3)}\n") + f.write(f"Scheduling times: {scheduling_delays}") + +def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_counter, requested_jobs, runtime_start, signature='', execution=False, print_logging=False): + if not os.path.exists(RESULTS_DIR): + os.mkdir(RESULTS_DIR) + + # Does not work. left here as reminder + if execution: + os.system("export LC_ALL=C.UTF-8") + os.system("export LANG=C.UTF-8") + + for run in range(repeats): + # Ensure complete cleanup from previous run + for f in [BASE, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR]: + if os.path.exists(f): + rmtree(f) + + file_base = os.path.join(BASE, 'testing') + pathlib.Path(file_base).mkdir(parents=True, exist_ok=True) + + runner_debug_stream = io.StringIO("") + + if execution: + runner = MeowRunner( + WatchdogMonitor(BASE, patterns, recipes, settletime=1), + PapermillHandler(), + LocalPythonConductor(), + print=runner_debug_stream, + logging=3 + ) + else: + runner = MeowRunner( + WatchdogMonitor(BASE, patterns, recipes, settletime=1), + PapermillHandler(), + DummyConductor(), + print=runner_debug_stream, + logging=3 + ) + +# meow.WorkflowRunner( +# VGRID, +# num_workers, +# patterns=patterns, +# recipes=recipes, +# daemon=True, +# start_workers=False, +# retro_active_jobs=False, +# print_logging=print_logging, +# file_logging=False, +# wait_time=1 +# ) + + runner.start() + + # Generate triggering files + first_filename, generation_duration = generate(files_count, file_base +"/file_") + + idle_loops = 0 + total_loops = 0 + messages = 0 + total_time = expected_job_count * 3 + if execution: + total_time = expected_job_count * 5 + while idle_loops < 10 and total_loops < total_time: + time.sleep(1) + runner_debug_stream.seek(0) + new_messages = len(runner_debug_stream.readlines()) + + if messages == new_messages: + idle_loops += 1 + else: + idle_loops = 0 + messages = new_messages + total_loops += 1 + + runner.stop() + + if execution: + jobs = os.listdir(DEFAULT_JOB_OUTPUT_DIR) + else: + jobs = os.listdir(DEFAULT_JOB_QUEUE_DIR) + + results_path = os.path.join(RESULTS_DIR, signature, str(expected_job_count), str(run), 'results.txt') + + cleanup(jobs, results_path, first_filename, generation_duration, execution=execution) + + print(f"Completed scheduling run {str(run + 1)} of {str(len(jobs))}/{str(expected_job_count)} jobs for '{signature}' {job_counter + expected_job_count*(run+1)}/{requested_jobs} ({str(round(time.time()-runtime_start, 3))}s)") + + collate_results(os.path.join(RESULTS_DIR, signature, str(expected_job_count))) diff --git a/benchmarking/srme.py b/benchmarking/srme.py new file mode 100644 index 0000000..e0ccfba --- /dev/null +++ b/benchmarking/srme.py @@ -0,0 +1,33 @@ + +from meow_base.patterns import FileEventPattern +from meow_base.recipes import getRecipeFromNotebook + +from shared import run_test, SRME + +def single_rule_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): + patterns = {} + pattern = FileEventPattern( + f"pattern_one", + f"testing/*", + "recipe_one", + "input" + ) + patterns[pattern.name] = pattern + + recipe = getRecipeFromNotebook("recipe_one", "test.ipynb") + + recipes = { + recipe.name: recipe + } + + run_test( + patterns, + recipes, + job_count, + job_count, + REPEATS, + job_counter, + requested_jobs, + runtime_start, + signature=SRME + ) diff --git a/benchmarking/srsep.py b/benchmarking/srsep.py new file mode 100644 index 0000000..582922e --- /dev/null +++ b/benchmarking/srsep.py @@ -0,0 +1,35 @@ + +from meow_base.patterns import FileEventPattern +from meow_base.recipes import getRecipeFromNotebook + +from meow_base.functionality.meow import create_parameter_sweep +from shared import run_test, SRSEP + +def single_rule_single_event_parallel(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): + patterns = {} + pattern = FileEventPattern( + f"pattern_one", + f"testing/*", + "recipe_one", + "input", + sweep=create_parameter_sweep("var", 1, job_count, 1) + ) + patterns[pattern.name] = pattern + + recipe = getRecipeFromNotebook("recipe_one", "test.ipynb") + + recipes = { + recipe.name: recipe + } + + run_test( + patterns, + recipes, + 1, + job_count, + REPEATS, + job_counter, + requested_jobs, + runtime_start, + signature=SRSEP + ) \ No newline at end of file diff --git a/benchmarking/srsps.py b/benchmarking/srsps.py new file mode 100644 index 0000000..40ad33a --- /dev/null +++ b/benchmarking/srsps.py @@ -0,0 +1,38 @@ + +from meow_base.patterns import FileEventPattern +from meow_base.recipes import getRecipeFromNotebook + +from shared import run_test, SRSES + +def single_rule_single_event_sequential(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start): + patterns = {} + pattern = FileEventPattern( + f"pattern_one", + f"testing/*", + "recipe_two", + "INPUT_FILE", + parameters={ + "MAX_COUNT":job_count + } + ) + patterns[pattern.name] = pattern + + recipe = getRecipeFromNotebook("recipe_two", "sequential.ipynb") + + recipes = { + recipe.name: recipe + } + + run_test( + patterns, + recipes, + 1, + job_count, + REPEATS, + job_counter, + requested_jobs, + runtime_start, + signature=SRSES, + execution=True, + print_logging=False + ) diff --git a/benchmarking/test.ipynb b/benchmarking/test.ipynb new file mode 100644 index 0000000..a7abefa --- /dev/null +++ b/benchmarking/test.ipynb @@ -0,0 +1,47 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mFailed to start the Kernel. \n", + "Failed to start the Kernel 'Python 3.6.9 64-bit'. \n", + "View Jupyter log for further details. Kernel has not been started" + ] + } + ], + "source": [ + "print('this is some outpug ')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + }, + "kernelspec": { + "display_name": "Python 3.6.9 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/conductors/__init__.py b/conductors/__init__.py index 13c5708..9fbe442 100644 --- a/conductors/__init__.py +++ b/conductors/__init__.py @@ -1,2 +1,2 @@ -from conductors.local_python_conductor import LocalPythonConductor +from .local_python_conductor import LocalPythonConductor diff --git a/conductors/local_python_conductor.py b/conductors/local_python_conductor.py index b5be437..6b87894 100644 --- a/conductors/local_python_conductor.py +++ b/conductors/local_python_conductor.py @@ -11,14 +11,14 @@ import shutil from datetime import datetime from typing import Any, Tuple, Dict -from core.base_conductor import BaseConductor -from core.correctness.meow import valid_job -from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ +from meow_base.core.base_conductor import BaseConductor +from meow_base.core.correctness.meow import valid_job +from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \ STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \ STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \ JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR -from core.correctness.validation import valid_dir_path -from functionality.file_io import make_dir, read_yaml, write_file, write_yaml +from meow_base.core.correctness.validation import valid_dir_path +from meow_base.functionality.file_io import make_dir, read_yaml, write_file, write_yaml class LocalPythonConductor(BaseConductor): def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/base_conductor.py b/core/base_conductor.py index a998f99..9745ce4 100644 --- a/core/base_conductor.py +++ b/core/base_conductor.py @@ -8,8 +8,8 @@ Author(s): David Marchant from typing import Any, Tuple, Dict -from core.correctness.vars import get_drt_imp_msg -from core.correctness.validation import check_implementation +from meow_base.core.correctness.vars import get_drt_imp_msg +from meow_base.core.correctness.validation import check_implementation class BaseConductor: diff --git a/core/base_handler.py b/core/base_handler.py index 285f159..b6654cf 100644 --- a/core/base_handler.py +++ b/core/base_handler.py @@ -8,8 +8,8 @@ Author(s): David Marchant from typing import Any, Tuple, Dict -from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS -from core.correctness.validation import check_implementation +from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS +from meow_base.core.correctness.validation import check_implementation class BaseHandler: diff --git a/core/base_monitor.py b/core/base_monitor.py index e8e709c..8f98035 100644 --- a/core/base_monitor.py +++ b/core/base_monitor.py @@ -9,12 +9,12 @@ Author(s): David Marchant from copy import deepcopy from typing import Union, Dict -from core.base_pattern import BasePattern -from core.base_recipe import BaseRecipe -from core.base_rule import BaseRule -from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS -from core.correctness.validation import check_implementation -from functionality.meow import create_rules +from meow_base.core.base_pattern import BasePattern +from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS +from meow_base.core.correctness.validation import check_implementation +from meow_base.functionality.meow import create_rules class BaseMonitor: diff --git a/core/base_pattern.py b/core/base_pattern.py index a93c8dc..ce909e4 100644 --- a/core/base_pattern.py +++ b/core/base_pattern.py @@ -10,9 +10,9 @@ import itertools from typing import Any, Union, Tuple, Dict, List -from core.correctness.vars import get_drt_imp_msg, \ +from meow_base.core.correctness.vars import get_drt_imp_msg, \ VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP -from core.correctness.validation import valid_string, check_type, \ +from meow_base.core.correctness.validation import valid_string, check_type, \ check_implementation, valid_dict diff --git a/core/base_recipe.py b/core/base_recipe.py index b91b452..7d50eeb 100644 --- a/core/base_recipe.py +++ b/core/base_recipe.py @@ -8,8 +8,8 @@ Author(s): David Marchant from typing import Any, Dict -from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS -from core.correctness.validation import valid_string, check_implementation +from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS +from meow_base.core.correctness.validation import valid_string, check_implementation class BaseRecipe: diff --git a/core/base_rule.py b/core/base_rule.py index ac4eb39..a87474b 100644 --- a/core/base_rule.py +++ b/core/base_rule.py @@ -10,11 +10,11 @@ from sys import modules from typing import Any if "BasePattern" not in modules: - from core.base_pattern import BasePattern + from meow_base.core.base_pattern import BasePattern if "BaseRecipe" not in modules: - from core.base_recipe import BaseRecipe -from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS -from core.correctness.validation import valid_string, check_type, \ + from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS +from meow_base.core.correctness.validation import valid_string, check_type, \ check_implementation diff --git a/core/correctness/meow.py b/core/correctness/meow.py index 8acf8fd..ba5b505 100644 --- a/core/correctness/meow.py +++ b/core/correctness/meow.py @@ -2,9 +2,9 @@ from datetime import datetime from typing import Any, Dict, Type -from core.base_rule import BaseRule -from core.correctness.validation import check_type -from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \ +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.validation import check_type +from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \ JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \ JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH diff --git a/core/correctness/validation.py b/core/correctness/validation.py index 38ae081..348c080 100644 --- a/core/correctness/validation.py +++ b/core/correctness/validation.py @@ -11,7 +11,7 @@ from os.path import sep, exists, isfile, isdir, dirname from typing import Any, _SpecialForm, Union, Type, Dict, List, \ get_origin, get_args -from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg +from meow_base.core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[], or_none:bool=False, hint:str="")->None: diff --git a/core/runner.py b/core/runner.py index 38d3cf5..271a2ea 100644 --- a/core/runner.py +++ b/core/runner.py @@ -14,16 +14,16 @@ from multiprocessing import Pipe from random import randrange from typing import Any, Union, Dict, List -from core.base_conductor import BaseConductor -from core.base_handler import BaseHandler -from core.base_monitor import BaseMonitor -from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \ +from meow_base.core.base_conductor import BaseConductor +from meow_base.core.base_handler import BaseHandler +from meow_base.core.base_monitor import BaseMonitor +from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \ VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR, \ EVENT_PATH -from core.correctness.validation import check_type, valid_list, valid_dir_path -from functionality.debug import setup_debugging, print_debug -from functionality.file_io import make_dir, read_yaml -from functionality.process_io import wait +from meow_base.core.correctness.validation import check_type, valid_list, valid_dir_path +from meow_base.functionality.debug import setup_debugging, print_debug +from meow_base.functionality.file_io import make_dir, read_yaml +from meow_base.functionality.process_io import wait class MeowRunner: @@ -112,6 +112,7 @@ class MeowRunner: if self._stop_mon_han_pipe[0] in ready: return else: + handled = False for from_monitor in self.from_monitors: if from_monitor in ready: # Read event from the monitor channel @@ -136,13 +137,25 @@ class MeowRunner: # If we've only one handler, use that if len(valid_handlers) == 1: handler = valid_handlers[0] + handled = True self.handle_event(handler, event) + break # If multiple handlers then randomly pick one - else: + elif len(valid_handlers) > 1: handler = valid_handlers[ randrange(len(valid_handlers)) ] + handled = True self.handle_event(handler, event) + break + + if not handled: + print_debug( + self._print_target, + self.debug_level, + "Could not determine handler for event.", + DEBUG_INFO + ) def run_handler_conductor_interaction(self)->None: """Function to be run in its own thread, to handle any inbound messages @@ -156,6 +169,7 @@ class MeowRunner: if self._stop_han_con_pipe[0] in ready: return else: + executed = False for from_handler in self.from_handlers: if from_handler in ready: # Read job directory from the handler channel @@ -188,14 +202,27 @@ class MeowRunner: # If we've only one conductor, use that if len(valid_conductors) == 1: conductor = valid_conductors[0] + executed = True self.execute_job(conductor, job_dir) + break # If multiple handlers then randomly pick one - else: + elif len(valid_conductors) > 1: conductor = valid_conductors[ randrange(len(valid_conductors)) ] + executed = True self.execute_job(conductor, job_dir) - + break + + # TODO determine something more useful to do here + if not executed: + print_debug( + self._print_target, + self.debug_level, + f"No conductor could be found for job {job_dir}", + DEBUG_INFO + ) + def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None: """Function for a given handler to handle a given event, without crashing the runner in the event of a problem.""" diff --git a/functionality/__init__.py b/functionality/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/functionality/debug.py b/functionality/debug.py index 0069c4d..56f4826 100644 --- a/functionality/debug.py +++ b/functionality/debug.py @@ -6,8 +6,8 @@ Author(s): David Marchant from typing import Any, Tuple -from core.correctness.validation import check_type -from core.correctness.vars import DEBUG_INFO, DEBUG_WARNING +from meow_base.core.correctness.validation import check_type +from meow_base.core.correctness.vars import DEBUG_INFO, DEBUG_WARNING def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]: diff --git a/functionality/file_io.py b/functionality/file_io.py index a2836e6..5107e31 100644 --- a/functionality/file_io.py +++ b/functionality/file_io.py @@ -11,7 +11,7 @@ from os import makedirs, remove, rmdir, walk from os.path import exists, isfile, join from typing import Any, Dict, List -from core.correctness.validation import valid_path +from meow_base.core.correctness.validation import valid_path def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False): diff --git a/functionality/hashing.py b/functionality/hashing.py index 7bcfe1a..d9855a5 100644 --- a/functionality/hashing.py +++ b/functionality/hashing.py @@ -6,8 +6,8 @@ Author(s): David Marchant from hashlib import sha256 -from core.correctness.vars import HASH_BUFFER_SIZE, SHA256 -from core.correctness.validation import check_type, valid_existing_file_path +from meow_base.core.correctness.vars import HASH_BUFFER_SIZE, SHA256 +from meow_base.core.correctness.validation import check_type, valid_existing_file_path def _get_file_sha256(file_path): sha256_hash = sha256() diff --git a/functionality/meow.py b/functionality/meow.py index f6c2657..96d8efe 100644 --- a/functionality/meow.py +++ b/functionality/meow.py @@ -8,16 +8,16 @@ from datetime import datetime from os.path import basename, dirname, relpath, splitext from typing import Any, Dict, Union, List -from core.base_pattern import BasePattern -from core.base_recipe import BaseRecipe -from core.base_rule import BaseRule -from core.correctness.validation import check_type, valid_dict, valid_list -from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ +from meow_base.core.base_pattern import BasePattern +from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.validation import check_type, valid_dict, valid_list +from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \ JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \ STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, SWEEP_START, \ SWEEP_STOP -from functionality.naming import generate_job_id, generate_rule_id +from meow_base.functionality.naming import generate_job_id, generate_rule_id # mig trigger keyword replacements KEYWORD_PATH = "{PATH}" @@ -208,7 +208,7 @@ def create_rule(pattern:BasePattern, recipe:BaseRecipe, # TODO fix me # Imported here to avoid circular imports at top of file - import rules + import meow_base.rules all_rules = { (r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__() } diff --git a/functionality/naming.py b/functionality/naming.py index ee245cb..6f063f1 100644 --- a/functionality/naming.py +++ b/functionality/naming.py @@ -7,7 +7,7 @@ Author(s): David Marchant from typing import List from random import SystemRandom -from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE +from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE #TODO Make this guaranteed unique diff --git a/functionality/parameterisation.py b/functionality/parameterisation.py index a23fefe..4c4e42d 100644 --- a/functionality/parameterisation.py +++ b/functionality/parameterisation.py @@ -10,7 +10,7 @@ from os import getenv from papermill.translators import papermill_translators from typing import Any, Dict, List -from core.correctness.validation import check_script, check_type +from meow_base.core.correctness.validation import check_script, check_type # Adapted from: https://github.com/rasmunk/notebook_parameterizer def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any], diff --git a/functionality/process_io.py b/functionality/process_io.py index 6cd4946..9d9c603 100644 --- a/functionality/process_io.py +++ b/functionality/process_io.py @@ -12,7 +12,7 @@ from multiprocessing.connection import Connection, wait as multi_wait if osName == 'nt': from multiprocessing.connection import PipeConnection from multiprocessing.queues import Queue -from core.correctness.vars import VALID_CHANNELS +from meow_base.core.correctness.vars import VALID_CHANNELS def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]: diff --git a/functionality/requirements.py b/functionality/requirements.py index 417b093..643d2f7 100644 --- a/functionality/requirements.py +++ b/functionality/requirements.py @@ -11,7 +11,7 @@ from os.path import basename from sys import version_info, prefix, base_prefix from typing import Any, Dict, List, Tuple, Union -from core.correctness.validation import check_type +from meow_base.core.correctness.validation import check_type REQUIREMENT_PYTHON = "python" REQ_PYTHON_MODULES = "modules" diff --git a/patterns/__init__.py b/patterns/__init__.py index 8867ea3..b892ed7 100644 --- a/patterns/__init__.py +++ b/patterns/__init__.py @@ -1,2 +1,2 @@ -from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor +from .file_event_pattern import FileEventPattern, WatchdogMonitor diff --git a/patterns/file_event_pattern.py b/patterns/file_event_pattern.py index a635ae6..6c82cda 100644 --- a/patterns/file_event_pattern.py +++ b/patterns/file_event_pattern.py @@ -18,19 +18,19 @@ from typing import Any, Union, Dict, List from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler -from core.base_recipe import BaseRecipe -from core.base_monitor import BaseMonitor -from core.base_pattern import BasePattern -from core.base_rule import BaseRule -from core.correctness.validation import check_type, valid_string, \ +from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.base_monitor import BaseMonitor +from meow_base.core.base_pattern import BasePattern +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.validation import check_type, valid_string, \ valid_dict, valid_list, valid_path, valid_dir_path -from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ +from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \ VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \ FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \ FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT -from functionality.debug import setup_debugging, print_debug -from functionality.hashing import get_file_hash -from functionality.meow import create_rule, create_watchdog_event +from meow_base.functionality.debug import setup_debugging, print_debug +from meow_base.functionality.hashing import get_file_hash +from meow_base.functionality.meow import create_rule, create_watchdog_event # Events that are monitored by default _DEFAULT_MASK = [ diff --git a/recipes/__init__.py b/recipes/__init__.py index 9f07fe8..b4d3dbf 100644 --- a/recipes/__init__.py +++ b/recipes/__init__.py @@ -1,4 +1,4 @@ -from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ - PapermillHandler -from recipes.python_recipe import PythonRecipe, PythonHandler \ No newline at end of file +from .jupyter_notebook_recipe import JupyterNotebookRecipe, PapermillHandler, \ + getRecipeFromNotebook +from .python_recipe import PythonRecipe, PythonHandler \ No newline at end of file diff --git a/recipes/jupyter_notebook_recipe.py b/recipes/jupyter_notebook_recipe.py index 5ad57e5..f6e2156 100644 --- a/recipes/jupyter_notebook_recipe.py +++ b/recipes/jupyter_notebook_recipe.py @@ -11,21 +11,21 @@ import sys from typing import Any, Tuple, Dict -from core.base_recipe import BaseRecipe -from core.base_handler import BaseHandler -from core.correctness.meow import valid_event -from core.correctness.validation import check_type, valid_string, \ - valid_dict, valid_path, valid_dir_path -from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ +from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.base_handler import BaseHandler +from meow_base.core.correctness.meow import valid_event +from meow_base.core.correctness.validation import check_type, valid_string, \ + valid_dict, valid_path, valid_dir_path, valid_existing_file_path +from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \ EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \ JOB_ID, WATCHDOG_BASE, META_FILE, \ PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \ EVENT_RULE, get_base_file -from functionality.debug import setup_debugging, print_debug -from functionality.file_io import make_dir, read_notebook, write_notebook, \ +from meow_base.functionality.debug import setup_debugging, print_debug +from meow_base.functionality.file_io import make_dir, read_notebook, write_notebook, \ write_yaml -from functionality.meow import create_job, replace_keywords +from meow_base.functionality.meow import create_job, replace_keywords class JupyterNotebookRecipe(BaseRecipe): @@ -182,20 +182,37 @@ class PapermillHandler(BaseHandler): # Send job directory, as actual definitons will be read from within it self.to_runner.send(job_dir) +#TODO test me +def getRecipeFromNotebook(name:str, notebook_filename:str, + parameters:Dict[str,Any]={}, requirements:Dict[str,Any]={} + )->JupyterNotebookRecipe: + valid_existing_file_path(notebook_filename, extension=".ipynb") + check_type(name, str, hint="getRecipeFromNotebook.name") + + notebook_code = read_notebook(notebook_filename) + + return JupyterNotebookRecipe( + name, + notebook_code, + parameters=parameters, + requirements=requirements, + source=notebook_filename + ) + # Papermill job execution code, to be run within the conductor def papermill_job_func(job_dir): # Requires own imports as will be run in its own execution environment import os import papermill from datetime import datetime - from core.correctness.vars import JOB_EVENT, JOB_ID, \ + from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \ EVENT_PATH, META_FILE, PARAMS_FILE, \ JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \ JOB_ERROR, STATUS_FAILED, get_job_file, \ get_result_file - from functionality.file_io import read_yaml, write_notebook, write_yaml - from functionality.hashing import get_file_hash - from functionality.parameterisation import parameterize_jupyter_notebook + from meow_base.functionality.file_io import read_yaml, write_notebook, write_yaml + from meow_base.functionality.hashing import get_file_hash + from meow_base.functionality.parameterisation import parameterize_jupyter_notebook # Identify job files diff --git a/recipes/python_recipe.py b/recipes/python_recipe.py index 4ed3433..7a9ac86 100644 --- a/recipes/python_recipe.py +++ b/recipes/python_recipe.py @@ -10,21 +10,21 @@ import sys from typing import Any, Tuple, Dict, List -from core.base_recipe import BaseRecipe -from core.base_handler import BaseHandler -from core.correctness.meow import valid_event -from core.correctness.validation import check_script, valid_string, \ +from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.base_handler import BaseHandler +from meow_base.core.correctness.meow import valid_event +from meow_base.core.correctness.validation import check_script, valid_string, \ valid_dict, valid_dir_path -from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ +from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \ DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \ EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \ JOB_ID, WATCHDOG_BASE, META_FILE, \ PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \ get_base_file -from functionality.debug import setup_debugging, print_debug -from functionality.file_io import make_dir, read_file_lines, write_file, \ +from meow_base.functionality.debug import setup_debugging, print_debug +from meow_base.functionality.file_io import make_dir, read_file_lines, write_file, \ write_yaml, lines_to_string -from functionality.meow import create_job, replace_keywords +from meow_base.functionality.meow import create_job, replace_keywords class PythonRecipe(BaseRecipe): @@ -180,14 +180,14 @@ def python_job_func(job_dir): import os from datetime import datetime from io import StringIO - from core.correctness.vars import JOB_EVENT, JOB_ID, \ + from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \ EVENT_PATH, META_FILE, PARAMS_FILE, \ JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \ JOB_ERROR, STATUS_FAILED, get_base_file, \ get_job_file, get_result_file - from functionality.file_io import read_yaml, write_yaml - from functionality.hashing import get_file_hash - from functionality.parameterisation import parameterize_python_script + from meow_base.functionality.file_io import read_yaml, write_yaml + from meow_base.functionality.hashing import get_file_hash + from meow_base.functionality.parameterisation import parameterize_python_script # Identify job files meta_file = os.path.join(job_dir, META_FILE) diff --git a/rules/__init__.py b/rules/__init__.py index 73a7d6c..6a5156c 100644 --- a/rules/__init__.py +++ b/rules/__init__.py @@ -1,3 +1,3 @@ -from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule -from rules.file_event_python_rule import FileEventPythonRule \ No newline at end of file +from .file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule +from .file_event_python_rule import FileEventPythonRule \ No newline at end of file diff --git a/rules/file_event_jupyter_notebook_rule.py b/rules/file_event_jupyter_notebook_rule.py index 12e6300..de1af8d 100644 --- a/rules/file_event_jupyter_notebook_rule.py +++ b/rules/file_event_jupyter_notebook_rule.py @@ -6,10 +6,10 @@ and JupyterNotebookRecipe. Author(s): David Marchant """ -from core.base_rule import BaseRule -from core.correctness.validation import check_type -from patterns.file_event_pattern import FileEventPattern -from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.validation import check_type +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe # TODO potentailly remove this and just invoke BaseRule directly, as does not # add any functionality other than some validation. diff --git a/rules/file_event_python_rule.py b/rules/file_event_python_rule.py index 600de46..48e4e78 100644 --- a/rules/file_event_python_rule.py +++ b/rules/file_event_python_rule.py @@ -6,10 +6,10 @@ and PythonRecipe. Author(s): David Marchant """ -from core.base_rule import BaseRule -from core.correctness.validation import check_type -from patterns.file_event_pattern import FileEventPattern -from recipes.python_recipe import PythonRecipe +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.validation import check_type +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.python_recipe import PythonRecipe # TODO potentailly remove this and just invoke BaseRule directly, as does not # add any functionality other than some validation. diff --git a/tests/shared.py b/tests/shared.py index eeddaa5..53d50a6 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -7,10 +7,10 @@ import os from distutils.dir_util import copy_tree -from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR -from functionality.file_io import make_dir, rmtree -from patterns import FileEventPattern -from recipes import JupyterNotebookRecipe +from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR +from meow_base.functionality.file_io import make_dir, rmtree +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe # testing TEST_DIR = "test_files" diff --git a/tests/test_base.py b/tests/test_base.py index b23cb7e..cff0f2d 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -3,14 +3,14 @@ import unittest from typing import Any, Union, Tuple, Dict -from core.base_conductor import BaseConductor -from core.base_handler import BaseHandler -from core.base_monitor import BaseMonitor -from core.base_pattern import BasePattern -from core.base_recipe import BaseRecipe -from core.base_rule import BaseRule -from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START -from patterns import FileEventPattern +from meow_base.core.base_conductor import BaseConductor +from meow_base.core.base_handler import BaseHandler +from meow_base.core.base_monitor import BaseMonitor +from meow_base.core.base_pattern import BasePattern +from meow_base.core.base_recipe import BaseRecipe +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START +from meow_base.patterns.file_event_pattern import FileEventPattern from shared import setup, teardown, valid_pattern_one, valid_recipe_one diff --git a/tests/test_conductors.py b/tests/test_conductors.py index bded7ee..10c4d43 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -5,21 +5,21 @@ import unittest from datetime import datetime from typing import Dict -from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \ +from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \ JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, JOB_EVENT, \ META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, JOB_PATTERN, \ STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \ JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \ EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file -from conductors import LocalPythonConductor -from functionality.file_io import read_file, read_yaml, write_file, \ +from meow_base.conductors import LocalPythonConductor +from meow_base.functionality.file_io import read_file, read_yaml, write_file, \ write_notebook, write_yaml, lines_to_string, make_dir -from functionality.hashing import get_file_hash -from functionality.meow import create_watchdog_event, create_job, create_rule -from patterns import FileEventPattern -from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ +from meow_base.functionality.hashing import get_file_hash +from meow_base.functionality.meow import create_watchdog_event, create_job, create_rule +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ papermill_job_func -from recipes.python_recipe import PythonRecipe, python_job_func +from meow_base.recipes.python_recipe import PythonRecipe, python_job_func from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \ TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, \ BAREBONES_PYTHON_SCRIPT, BAREBONES_NOTEBOOK diff --git a/tests/test_functionality.py b/tests/test_functionality.py index c2320c6..f709eec 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -11,34 +11,34 @@ from sys import prefix, base_prefix from time import sleep from typing import Dict -from core.base_rule import BaseRule -from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ +from meow_base.core.base_rule import BaseRule +from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \ WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \ PYTHON_FUNC, JOB_ID, JOB_EVENT, \ JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL -from functionality.debug import setup_debugging -from functionality.file_io import lines_to_string, make_dir, read_file, \ +from meow_base.functionality.debug import setup_debugging +from meow_base.functionality.file_io import lines_to_string, make_dir, read_file, \ read_file_lines, read_notebook, read_yaml, rmtree, write_file, \ write_notebook, write_yaml -from functionality.hashing import get_file_hash -from functionality.meow import create_event, create_job, create_rule, \ +from meow_base.functionality.hashing import get_file_hash +from meow_base.functionality.meow import create_event, create_job, create_rule, \ create_rules, create_watchdog_event, replace_keywords, \ create_parameter_sweep, \ KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \ KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \ KEYWORD_REL_PATH -from functionality.naming import _generate_id -from functionality.parameterisation import parameterize_jupyter_notebook, \ +from meow_base.functionality.naming import _generate_id +from meow_base.functionality.parameterisation import parameterize_jupyter_notebook, \ parameterize_python_script -from functionality.process_io import wait -from functionality.requirements import create_python_requirements, \ +from meow_base.functionality.process_io import wait +from meow_base.functionality.requirements import create_python_requirements, \ check_requirements, \ REQUIREMENT_PYTHON, REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, \ REQ_PYTHON_VERSION -from patterns import FileEventPattern -from recipes import JupyterNotebookRecipe +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \ valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \ COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT @@ -594,7 +594,7 @@ class MeowTests(unittest.TestCase): self.assertEqual(len(rules), 0) - # Test that create_rules creates rules from patterns and recipes + # Test that create_rules creates rules from meow_base.patterns and recipes def testCreateRulesPatternsAndRecipesDicts(self)->None: patterns = { valid_pattern_one.name: valid_pattern_one, diff --git a/tests/test_patterns.py b/tests/test_patterns.py index b6f9473..bff1519 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -5,13 +5,13 @@ import unittest from multiprocessing import Pipe -from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \ +from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \ EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \ SWEEP_JUMP, SWEEP_STOP -from functionality.file_io import make_dir -from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \ +from meow_base.functionality.file_io import make_dir +from meow_base.patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \ _DEFAULT_MASK -from recipes import JupyterNotebookRecipe +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE diff --git a/tests/test_recipes.py b/tests/test_recipes.py index e854a9e..29adac2 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -6,23 +6,23 @@ import unittest from multiprocessing import Pipe from typing import Dict -from core.correctness.meow import valid_job -from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ +from meow_base.core.correctness.meow import valid_job +from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \ EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \ JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \ META_FILE, JOB_ERROR, \ PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \ get_base_file, get_job_file, get_result_file -from functionality.file_io import lines_to_string, make_dir, read_yaml, \ +from meow_base.functionality.file_io import lines_to_string, make_dir, read_yaml, \ write_file, write_notebook, write_yaml -from functionality.hashing import get_file_hash -from functionality.meow import create_job, create_rules, create_rule, \ +from meow_base.functionality.hashing import get_file_hash +from meow_base.functionality.meow import create_job, create_rules, create_rule, \ create_watchdog_event -from patterns.file_event_pattern import FileEventPattern -from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler, papermill_job_func -from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func -from rules import FileEventJupyterNotebookRule, FileEventPythonRule +from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func +from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \ COMPLETE_PYTHON_SCRIPT, TEST_JOB_QUEUE, TEST_MONITOR_BASE, \ TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK diff --git a/tests/test_rules.py b/tests/test_rules.py index c147597..e3d48ba 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -1,9 +1,9 @@ import unittest -from patterns.file_event_pattern import FileEventPattern -from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe -from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule +from meow_base.patterns.file_event_pattern import FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe +from meow_base.rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule from shared import setup, teardown, BAREBONES_NOTEBOOK class CorrectnessTests(unittest.TestCase): diff --git a/tests/test_runner.py b/tests/test_runner.py index d532242..8463f78 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -8,21 +8,21 @@ from random import shuffle from shutil import copy from time import sleep -from core.base_conductor import BaseConductor -from core.base_handler import BaseHandler -from core.base_monitor import BaseMonitor -from conductors import LocalPythonConductor -from core.correctness.vars import get_result_file, \ +from meow_base.core.base_conductor import BaseConductor +from meow_base.core.base_handler import BaseHandler +from meow_base.core.base_monitor import BaseMonitor +from meow_base.conductors import LocalPythonConductor +from meow_base.core.correctness.vars import get_result_file, \ JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME -from core.runner import MeowRunner -from functionality.file_io import make_dir, read_file, read_notebook, \ +from meow_base.core.runner import MeowRunner +from meow_base.functionality.file_io import make_dir, read_file, read_notebook, \ read_yaml, write_file, lines_to_string -from functionality.meow import create_parameter_sweep -from functionality.requirements import create_python_requirements -from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern -from recipes.jupyter_notebook_recipe import PapermillHandler, \ +from meow_base.functionality.meow import create_parameter_sweep +from meow_base.functionality.requirements import create_python_requirements +from meow_base.patterns.file_event_pattern import WatchdogMonitor, FileEventPattern +from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler, \ JupyterNotebookRecipe -from recipes.python_recipe import PythonHandler, PythonRecipe +from meow_base.recipes.python_recipe import PythonHandler, PythonRecipe from shared import setup, teardown, backup_before_teardown, \ TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, MAKER_RECIPE, \ APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, FILTER_RECIPE, \ @@ -916,10 +916,6 @@ class MeowTests(unittest.TestCase): output = read_file(os.path.join(output_path)) self.assertEqual(output, "12505000.0") - - - - def testSelfModifyingAnalysis(self)->None: maker_pattern = FileEventPattern( "maker_pattern", diff --git a/tests/test_validation.py b/tests/test_validation.py index c58ad83..a1a6aef 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -5,16 +5,16 @@ import os from datetime import datetime from typing import Any, Union -from core.correctness.meow import valid_event, valid_job, valid_watchdog_event -from core.correctness.validation import check_type, check_implementation, \ +from meow_base.core.correctness.meow import valid_event, valid_job, valid_watchdog_event +from meow_base.core.correctness.validation import check_type, check_implementation, \ valid_string, valid_dict, valid_list, valid_existing_file_path, \ valid_dir_path, valid_non_existing_path, check_callable -from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \ +from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \ EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \ JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \ WATCHDOG_HASH -from functionality.file_io import make_dir -from functionality.meow import create_rule +from meow_base.functionality.file_io import make_dir +from meow_base.functionality.meow import create_rule from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \ valid_recipe_one