reformatted imports to work better on other machines, plus added benchmarking to project

.gitignore

@@ -58,6 +58,10 @@ tests/test_data
 tests/job_output
 tests/job_queue
 tests/Backup*
+benchmarking/benchmark_base
+benchmarking/job_output
+benchmarking/job_queue
+benchmarking/result*
 
 # hdf5
 *.h5

benchmarking/mrme.py

@@ -0,0 +1,34 @@
+
+from meow_base.patterns import FileEventPattern
+from meow_base.recipes import getRecipeFromNotebook
+
+from shared import run_test, MRME
+
+def multiple_rules_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
+    patterns = {}
+    for i in range(job_count):
+        pattern = FileEventPattern(
+            f"pattern_{i}",
+            f"testing/file_{i}.txt",
+            "recipe_one",
+            "input"
+        )
+        patterns[pattern.name] = pattern
+
+    recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
+    
+    recipes = {
+        recipe.name: recipe
+    }
+
+    run_test(
+        patterns, 
+        recipes, 
+        job_count, 
+        job_count,
+        REPEATS, 
+        job_counter,
+        requested_jobs,
+        runtime_start,
+        signature=MRME
+    )

benchmarking/mrse.py

@@ -0,0 +1,34 @@
+
+from meow_base.patterns import FileEventPattern
+from meow_base.recipes import getRecipeFromNotebook
+
+from shared import run_test, MRSE
+
+def multiple_rules_single_event(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
+    patterns = {}
+    for i in range(job_count):
+        pattern = FileEventPattern(
+            f"pattern_{i}",
+            f"testing/*",
+            "recipe_one",
+            "input"
+        )
+        patterns[pattern.name] = pattern
+
+    recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
+    
+    recipes = {
+        recipe.name: recipe
+    }
+
+    run_test(
+        patterns, 
+        recipes, 
+        1, 
+        job_count,
+        REPEATS, 
+        job_counter,
+        requested_jobs,
+        runtime_start,
+        signature=MRSE
+    )

benchmarking/run_all.py

@@ -0,0 +1,175 @@
+
+import matplotlib.pyplot as pyplot
+import numpy
+import sys
+import time
+import os
+
+from shared import JOBS_COUNTS, REPEATS, TESTS, MRME, MRSE, SRME, SRSEP, SRSES, RESULTS_DIR, BASE, GRAPH_FILENAME
+from mrme import multiple_rules_multiple_events
+from mrse import multiple_rules_single_event
+from srme import single_rule_multiple_events
+from srsep import single_rule_single_event_parallel
+from srsps import single_rule_single_event_sequential
+
+from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
+from meow_base.functionality.file_io import rmtree
+
+LINE_KEYS = {
+    SRSES: ('x','#a1467e'),
+    SRME: ('.','#896cff'),
+    MRME: ('d','#5983b0'),
+    MRSE: ('P','#ff6cbe'),
+    SRSEP: ('*','#3faf46'),
+}
+
+
+def run_tests():
+    rmtree(RESULTS_DIR)
+
+    requested_jobs=0
+    for job_count in JOBS_COUNTS:
+        requested_jobs += job_count * REPEATS * len(TESTS)
+    print(f"requested_jobs: {requested_jobs}")
+
+    runtime_start=time.time()
+  
+    job_counter=0
+    for job_count in JOBS_COUNTS:
+        for test in TESTS:
+            if test == MRME:
+                multiple_rules_multiple_events(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
+                job_counter += job_count * REPEATS
+
+            elif test == MRSE:
+                multiple_rules_single_event(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
+                job_counter += job_count * REPEATS
+
+            elif test == SRME:
+                single_rule_multiple_events(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
+                job_counter += job_count * REPEATS
+
+            elif test == SRSEP:
+                single_rule_single_event_parallel(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
+                job_counter += job_count * REPEATS
+
+            elif test == SRSES:
+                single_rule_single_event_sequential(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
+                job_counter += job_count * REPEATS
+
+    print(f"All tests completed in: {str(time.time()-runtime_start)}")
+
+def get_meow_graph(results_dir):
+    lines = []
+
+    for run_type in os.listdir(results_dir):
+        #if run_type == 'single_Pattern_single_file_sequential':
+        #        continue
+
+#       lines.append((f'scheduling {run_type}', [], 'solid'))
+        lines.append((run_type, [], 'solid'))
+        run_type_path = os.path.join(results_dir, run_type)
+
+        for job_count in os.listdir(run_type_path):
+            results_path = os.path.join(run_type_path, job_count, 'results.txt')
+            with open(results_path, 'r') as f_in:
+                data = f_in.readlines()
+
+            scheduling_duration = 0
+            for line in data:
+                if "Average schedule time: " in line:
+                    scheduling_duration = float(line.replace("Average schedule time: ", ''))
+    
+            lines[-1][1].append((job_count, scheduling_duration))
+            lines[-1][1].sort(key=lambda y: float(y[0]))
+
+    return lines
+
+def make_plot(lines, graph_path, title, logged):
+    w = 10
+    h = 4
+    linecount = 0
+    columns = 1
+
+    pyplot.figure(figsize=(w, h))
+    for l in range(len(lines)):
+        x_values = numpy.asarray([float(i[0]) for i in lines[l][1]])
+        y_values = numpy.asarray([float(i[1]) for i in lines[l][1]])
+
+        # Remove this check to always display lines
+        if lines[l][2] == 'solid':
+            pyplot.plot(x_values, y_values, label=lines[l][0], linestyle=lines[l][2], marker=LINE_KEYS[lines[l][0]][0], color=LINE_KEYS[lines[l][0]][1])
+            linecount += 1
+
+    columns = int(linecount/3) + 1
+                    
+    pyplot.xlabel("Number of jobs scheduled")
+    pyplot.ylabel("Time taken (seconds)")
+    pyplot.title(title)
+
+    handles, labels = pyplot.gca().get_legend_handles_labels()
+    #    legend_order = [2, 4, 0, 1, 3]
+    #    pyplot.legend([handles[i] for i in legend_order], [labels[i] for i in legend_order])
+
+    pyplot.legend(ncol=columns, prop={'size': 12})
+    if logged:
+        pyplot.yscale('log')
+
+    x_ticks = []
+    for tick in x_values:
+        label = int(tick)
+        if tick <= 100 and tick % 20 == 0:
+            label = f"\n{int(tick)}"
+        x_ticks.append(label)
+
+    pyplot.xticks(x_values, x_ticks)
+
+    pyplot.savefig(graph_path, format='pdf', bbox_inches='tight')
+
+def make_both_plots(lines, path, title, log=True):
+    make_plot(lines, path, title, False)
+    if log:
+        logged_path = path[:path.index(".pdf")] + "_logged" + path[path.index(".pdf"):]
+        make_plot(lines, logged_path, title, True)
+
+
+def make_graphs():
+    lines = get_meow_graph(RESULTS_DIR)
+
+    make_both_plots(lines, "result.pdf", "MiG scheduling overheads on the Threadripper")
+
+    average_lines = []
+    all_delta_lines = []
+    no_spsfs_delta_lines = []
+    for line_signature, line_values, lines_style in lines:
+        if lines_style == 'solid':
+            averages = [(i, v/float(i)) for i, v in line_values]
+            average_lines.append((line_signature, averages, lines_style))
+
+            if line_signature not in ["total single_Pattern_single_file_sequential", "scheduling single_Pattern_single_file_sequential_jobs", "SPSFS"]:
+                deltas = []
+                for i in range(len(line_values)-1):
+                    deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) )
+                no_spsfs_delta_lines.append((line_signature, deltas, lines_style))
+            deltas = []
+            for i in range(len(line_values)-1):
+                deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) )
+            all_delta_lines.append((line_signature, deltas, lines_style))
+
+
+    make_both_plots(average_lines, "result_averaged.pdf", "Per-job MiG scheduling overheads on the Threadripper")
+
+    make_both_plots(all_delta_lines, "result_deltas.pdf", "Difference in per-job MiG scheduling overheads on the Threadripper", log=False)
+
+if __name__ == '__main__':
+    try:
+        run_tests()
+        make_graphs()
+        rmtree(DEFAULT_JOB_QUEUE_DIR)
+        rmtree(DEFAULT_JOB_OUTPUT_DIR)
+        rmtree(BASE)
+    except KeyboardInterrupt as ki:
+        try:
+            sys.exit(1)
+        except SystemExit:
+            os._exit(1)

benchmarking/sequential.ipynb

@@ -0,0 +1,69 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "INPUT_FILE = 'file_0.txt'\n",
+    "MAX_COUNT = 100"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(INPUT_FILE, 'r') as f:\n",
+    "    data = int(f.read())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\"read in: \"+ str(data))\n",
+    "print(\"writing out: \"+ str(data+1))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if data+1 < MAX_COUNT:\n",
+    "    with open(INPUT_FILE.replace(str(data), str(data+1)), 'w') as f:\n",
+    "        f.write(str(data+1))"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

benchmarking/shared.py

@@ -0,0 +1,225 @@
+
+import datetime
+import io
+import os
+import pathlib
+import time
+import yaml
+
+from typing import Any, Dict, Tuple
+
+from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
+from meow_base.core.runner import MeowRunner
+from meow_base.patterns.file_event_pattern import WatchdogMonitor
+from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler
+from meow_base.conductors import LocalPythonConductor
+from meow_base.functionality.file_io import rmtree
+
+RESULTS_DIR = "results"
+BASE = "benchmark_base"
+GRAPH_FILENAME = "graph.pdf"
+REPEATS = 10
+JOBS_COUNTS = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, 250, 300, 400, 500]
+
+SRME = "single_rule_multiple_events"
+MRSE = "multiple_rules_single_event"
+SRSEP = "single_rule_single_event_parallel"
+MRME = "multiple_rules_multiple_events"
+SRSES = "single_rule_single_event_sequential"
+
+TESTS = [ 
+    SRME, 
+    MRSE,
+    SRSEP, 
+    MRME,
+    # This test will take approx 90% of total time
+    SRSES 
+]
+
+
+class DummyConductor(LocalPythonConductor):
+    def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
+        return False, ">:("
+
+
+def datetime_to_timestamp(date_time_obj):
+    return time.mktime(date_time_obj.timetuple()) + float(date_time_obj.microsecond)/1000000 
+
+def generate(file_count, file_path, file_type='.txt'):
+    first_filename = ''
+    start = time.time()
+    for i in range(int(file_count)):
+        filename = file_path + str(i) + file_type
+        if not first_filename:
+            first_filename = filename
+        with open(filename, 'w') as f:
+            f.write('0')
+    return first_filename, time.time() - start
+
+def cleanup(jobs, file_out, base_time, gen_time, execution=False):
+    if not jobs:
+        return
+
+    job_timestamps = []
+    for job in jobs:
+        if execution:
+            with open(f"{DEFAULT_JOB_OUTPUT_DIR}/{job}/job.yml", 'r') as f_in:
+                data = yaml.load(f_in, Loader=yaml.Loader)
+        else:
+            with open(f"{DEFAULT_JOB_QUEUE_DIR}/{job}/job.yml", 'r') as f_in:
+                data = yaml.load(f_in, Loader=yaml.Loader)
+        create_datetime = data['create']
+        create_timestamp = datetime_to_timestamp(create_datetime)
+        job_timestamps.append((create_timestamp, create_datetime))
+
+    job_timestamps.sort(key=lambda y: int(y[0]))
+
+    first = job_timestamps[0]
+    last = job_timestamps[-1]
+
+    #dt = datetime.datetime.fromtimestamp(os.path.getctime(base_time), datetime.timezone(datetime.timedelta(hours=0)))
+    dt = datetime.datetime.fromtimestamp(os.path.getctime(base_time))
+
+#    if execution:
+#        queue_times = []
+#        execution_times = []
+#        for j in jobs:
+#            mrsl_dict = load(os.path.join(mrsl_dir, j))#
+#
+#            queue_times.append(time.mktime(mrsl_dict['EXECUTING_TIMESTAMP']) - time.mktime(mrsl_dict['QUEUED_TIMESTAMP']))
+#            execution_times.append(time.mktime(mrsl_dict['FINISHED_TIMESTAMP']) - time.mktime(mrsl_dict['EXECUTING_TIMESTAMP']))
+    pathlib.Path(os.path.dirname(file_out)).mkdir(parents=True, exist_ok=True)
+    with open(file_out, 'w') as f_out:
+        f_out.write("Job count: "+ str(len(jobs)) +"\n")
+        f_out.write("Generation time: "+ str(round(gen_time, 5)) +"\n")
+        f_out.write("First trigger: "+ str(dt) +"\n")
+        f_out.write("First scheduling datetime: "+ str(first[1]) +"\n")
+        f_out.write("Last scheduling datetime: "+ str(last[1]) +"\n")
+        f_out.write("First scheduling unixtime: "+ str(first[0]) +"\n")
+        f_out.write("First scheduling unixtime: "+ str(last[0]) +"\n")
+        f_out.write("Scheduling difference (seconds): "+ str(round(last[0] - first[0], 3)) +"\n")
+        f_out.write("Initial scheduling delay (seconds): "+ str(round(first[0] - os.path.getctime(base_time), 3)) +"\n")
+        total_time = round(last[0] - os.path.getctime(base_time), 3)
+        f_out.write("Total scheduling delay (seconds): "+ str(total_time) +"\n")
+
+#        if execution:
+#            f_out.write("Average execution time (seconds): "+ str(round(mean(execution_times), 3)) +"\n")
+#            f_out.write("Max execution time (seconds): "+ str(round(max(execution_times), 3)) +"\n")
+#            f_out.write("Min execution time (seconds): "+ str(round(min(execution_times), 3)) +"\n")
+
+#            f_out.write("Average queueing delay (seconds): "+ str(round(mean(queue_times), 3)) +"\n")
+#            f_out.write("Max queueing delay (seconds): "+ str(round(max(queue_times), 3)) +"\n")
+#            f_out.write("Min queueing delay (seconds): "+ str(round(min(queue_times), 3)) +"\n")
+
+#            queue_times.remove(max(queue_times))
+#            f_out.write("Average excluded queueing delay (seconds): "+ str(round(mean(queue_times), 3)) +"\n")
+
+    return total_time
+
+def mean(l):
+    return sum(l)/len(l)
+
+def collate_results(base_results_dir):
+
+    scheduling_delays = []
+
+    for run in os.listdir(base_results_dir):
+        if run != 'results.txt':
+            with open(os.path.join(base_results_dir, run, 'results.txt'), 'r') as f:
+                d = f.readlines()
+
+                for l in d:
+                    if "Total scheduling delay (seconds): " in l:
+                        scheduling_delays.append(float(l.replace("Total scheduling delay (seconds): ", '')))
+
+    with open(os.path.join(base_results_dir, 'results.txt'), 'w') as f:
+        f.write(f"Average schedule time: {round(mean(scheduling_delays), 3)}\n")
+        f.write(f"Scheduling times: {scheduling_delays}")
+
+def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_counter, requested_jobs, runtime_start, signature='', execution=False, print_logging=False):
+    if not os.path.exists(RESULTS_DIR):
+        os.mkdir(RESULTS_DIR)
+
+    # Does not work. left here as reminder
+    if execution:
+        os.system("export LC_ALL=C.UTF-8")
+        os.system("export LANG=C.UTF-8")
+
+    for run in range(repeats):
+        # Ensure complete cleanup from previous run
+        for f in [BASE, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR]:
+            if os.path.exists(f):
+                rmtree(f)
+
+        file_base = os.path.join(BASE, 'testing')
+        pathlib.Path(file_base).mkdir(parents=True, exist_ok=True)
+
+        runner_debug_stream = io.StringIO("")
+
+        if execution:
+            runner = MeowRunner(
+                WatchdogMonitor(BASE, patterns, recipes, settletime=1),
+                PapermillHandler(),
+                LocalPythonConductor(),
+                print=runner_debug_stream,
+                logging=3
+            )
+        else:
+            runner = MeowRunner(
+                WatchdogMonitor(BASE, patterns, recipes, settletime=1),
+                PapermillHandler(),
+                DummyConductor(),
+                print=runner_debug_stream,
+                logging=3
+            )
+        
+#        meow.WorkflowRunner(
+#            VGRID,
+#            num_workers,
+#            patterns=patterns,
+#            recipes=recipes,
+#            daemon=True,
+#            start_workers=False,
+#            retro_active_jobs=False,
+#            print_logging=print_logging,
+#            file_logging=False,
+#            wait_time=1
+#        )
+
+        runner.start()
+
+        # Generate triggering files
+        first_filename, generation_duration = generate(files_count, file_base +"/file_")
+
+        idle_loops = 0
+        total_loops = 0
+        messages = 0
+        total_time = expected_job_count * 3
+        if execution:
+            total_time = expected_job_count * 5
+        while idle_loops < 10 and total_loops < total_time:
+            time.sleep(1)
+            runner_debug_stream.seek(0)
+            new_messages = len(runner_debug_stream.readlines())
+
+            if messages == new_messages:               
+                idle_loops += 1
+            else:
+                idle_loops = 0
+                messages = new_messages
+            total_loops += 1
+
+        runner.stop()
+
+        if execution:
+            jobs = os.listdir(DEFAULT_JOB_OUTPUT_DIR)
+        else:
+            jobs = os.listdir(DEFAULT_JOB_QUEUE_DIR)
+
+        results_path = os.path.join(RESULTS_DIR, signature, str(expected_job_count), str(run), 'results.txt')
+
+        cleanup(jobs, results_path, first_filename, generation_duration, execution=execution)
+
+        print(f"Completed scheduling run {str(run + 1)} of {str(len(jobs))}/{str(expected_job_count)} jobs for '{signature}' {job_counter + expected_job_count*(run+1)}/{requested_jobs} ({str(round(time.time()-runtime_start, 3))}s)")
+
+    collate_results(os.path.join(RESULTS_DIR, signature, str(expected_job_count)))

benchmarking/srme.py

@@ -0,0 +1,33 @@
+
+from meow_base.patterns import FileEventPattern
+from meow_base.recipes import getRecipeFromNotebook
+
+from shared import run_test, SRME
+
+def single_rule_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
+    patterns = {}
+    pattern = FileEventPattern(
+        f"pattern_one",
+        f"testing/*",
+        "recipe_one",
+        "input"
+    )
+    patterns[pattern.name] = pattern
+
+    recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
+    
+    recipes = {
+        recipe.name: recipe
+    }
+
+    run_test(
+        patterns, 
+        recipes, 
+        job_count, 
+        job_count,
+        REPEATS, 
+        job_counter,
+        requested_jobs,
+        runtime_start,
+        signature=SRME
+    )

benchmarking/srsep.py

@@ -0,0 +1,35 @@
+
+from meow_base.patterns import FileEventPattern
+from meow_base.recipes import getRecipeFromNotebook
+
+from meow_base.functionality.meow import create_parameter_sweep
+from shared import run_test, SRSEP
+
+def single_rule_single_event_parallel(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
+    patterns = {}
+    pattern = FileEventPattern(
+        f"pattern_one",
+        f"testing/*",
+        "recipe_one",
+        "input",
+        sweep=create_parameter_sweep("var", 1, job_count, 1)
+    )
+    patterns[pattern.name] = pattern
+
+    recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
+    
+    recipes = {
+        recipe.name: recipe
+    }
+
+    run_test(
+        patterns, 
+        recipes, 
+        1, 
+        job_count,
+        REPEATS, 
+        job_counter,
+        requested_jobs,
+        runtime_start,
+        signature=SRSEP
+    )

benchmarking/srsps.py

@@ -0,0 +1,38 @@
+
+from meow_base.patterns import FileEventPattern
+from meow_base.recipes import getRecipeFromNotebook
+
+from shared import run_test, SRSES
+
+def single_rule_single_event_sequential(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
+    patterns = {}
+    pattern = FileEventPattern(
+        f"pattern_one",
+        f"testing/*",
+        "recipe_two",
+        "INPUT_FILE",
+        parameters={
+            "MAX_COUNT":job_count
+        }
+    )
+    patterns[pattern.name] = pattern
+
+    recipe = getRecipeFromNotebook("recipe_two", "sequential.ipynb")
+    
+    recipes = {
+        recipe.name: recipe
+    }
+
+    run_test(
+        patterns, 
+        recipes, 
+        1, 
+        job_count,
+        REPEATS, 
+        job_counter,
+        requested_jobs,
+        runtime_start,
+        signature=SRSES,
+        execution=True,
+        print_logging=False
+    )

benchmarking/test.ipynb

@@ -0,0 +1,47 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mFailed to start the Kernel. \n",
+      "Failed to start the Kernel 'Python 3.6.9 64-bit'. \n",
+      "View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details. Kernel has not been started"
+     ]
+    }
+   ],
+   "source": [
+    "print('this is some outpug ')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.6.9 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.6.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

conductors/__init__.py

@@ -1,2 +1,2 @@
 
-from conductors.local_python_conductor import LocalPythonConductor
+from .local_python_conductor import LocalPythonConductor

conductors/local_python_conductor.py

@@ -11,14 +11,14 @@ import shutil
 from datetime import datetime
 from typing import Any, Tuple, Dict
 
-from core.base_conductor import BaseConductor
+from meow_base.core.base_conductor import BaseConductor
-from core.correctness.meow import valid_job
+from meow_base.core.correctness.meow import valid_job
-from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
+from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
     STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
     STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
     JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
-from core.correctness.validation import valid_dir_path
+from meow_base.core.correctness.validation import valid_dir_path
-from functionality.file_io import make_dir, read_yaml, write_file, write_yaml
+from meow_base.functionality.file_io import make_dir, read_yaml, write_file, write_yaml
 
 class LocalPythonConductor(BaseConductor):
     def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR, 

core/base_conductor.py

@@ -8,8 +8,8 @@ Author(s): David Marchant
 
 from typing import Any, Tuple, Dict
 
-from core.correctness.vars import get_drt_imp_msg
+from meow_base.core.correctness.vars import get_drt_imp_msg
-from core.correctness.validation import check_implementation
+from meow_base.core.correctness.validation import check_implementation
 
 
 class BaseConductor:

core/base_handler.py

@@ -8,8 +8,8 @@ Author(s): David Marchant
 
 from typing import Any, Tuple, Dict
 
-from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
+from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
-from core.correctness.validation import check_implementation
+from meow_base.core.correctness.validation import check_implementation
 
 
 class BaseHandler:

core/base_monitor.py

@@ -9,12 +9,12 @@ Author(s): David Marchant
 from copy import deepcopy
 from typing import Union, Dict
 
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
+from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
-from core.correctness.validation import check_implementation
+from meow_base.core.correctness.validation import check_implementation
-from functionality.meow import create_rules
+from meow_base.functionality.meow import create_rules
 
 
 class BaseMonitor:

core/base_pattern.py

@@ -10,9 +10,9 @@ import itertools
 
 from typing import Any, Union, Tuple, Dict, List
 
-from core.correctness.vars import get_drt_imp_msg, \
+from meow_base.core.correctness.vars import get_drt_imp_msg, \
     VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
-from core.correctness.validation import valid_string, check_type, \
+from meow_base.core.correctness.validation import valid_string, check_type, \
     check_implementation, valid_dict
 
 

core/base_recipe.py

@@ -8,8 +8,8 @@ Author(s): David Marchant
 
 from typing import Any, Dict
 
-from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
+from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
-from core.correctness.validation import valid_string, check_implementation
+from meow_base.core.correctness.validation import valid_string, check_implementation
 
 
 class BaseRecipe:

core/base_rule.py

@@ -10,11 +10,11 @@ from sys import modules
 from typing import Any
 
 if "BasePattern" not in modules:
-    from core.base_pattern import BasePattern
+    from meow_base.core.base_pattern import BasePattern
 if "BaseRecipe" not in modules:
-    from core.base_recipe import BaseRecipe
+    from meow_base.core.base_recipe import BaseRecipe
-from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
+from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
-from core.correctness.validation import valid_string, check_type, \
+from meow_base.core.correctness.validation import valid_string, check_type, \
     check_implementation
 
 

core/correctness/meow.py

@@ -2,9 +2,9 @@
 from datetime import datetime
 from typing import Any, Dict, Type
 
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.validation import check_type
+from meow_base.core.correctness.validation import check_type
-from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
+from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
     JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \
     JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
 

core/correctness/validation.py

@@ -11,7 +11,7 @@ from os.path import sep, exists, isfile, isdir, dirname
 from typing import Any, _SpecialForm, Union, Type, Dict, List, \
     get_origin, get_args
 
-from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
+from meow_base.core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
 
 def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[], 
         or_none:bool=False, hint:str="")->None:

core/runner.py

@@ -14,16 +14,16 @@ from multiprocessing import Pipe
 from random import randrange
 from typing import Any, Union, Dict, List
 
-from core.base_conductor import BaseConductor
+from meow_base.core.base_conductor import BaseConductor
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.base_monitor import BaseMonitor
+from meow_base.core.base_monitor import BaseMonitor
-from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
+from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
     VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR, \
     EVENT_PATH
-from core.correctness.validation import check_type, valid_list, valid_dir_path
+from meow_base.core.correctness.validation import check_type, valid_list, valid_dir_path
-from functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.debug import setup_debugging, print_debug
-from functionality.file_io import make_dir, read_yaml
+from meow_base.functionality.file_io import make_dir, read_yaml
-from functionality.process_io import wait
+from meow_base.functionality.process_io import wait
 
 
 class MeowRunner:
@@ -112,6 +112,7 @@ class MeowRunner:
             if self._stop_mon_han_pipe[0] in ready:
                 return
             else:
+                handled = False
                 for from_monitor in self.from_monitors:
                     if from_monitor in ready:
                         # Read event from the monitor channel
@@ -136,13 +137,25 @@ class MeowRunner:
                         # If we've only one handler, use that
                         if len(valid_handlers) == 1:
                             handler = valid_handlers[0]
+                            handled = True
                             self.handle_event(handler, event)
+                            break
                         # If multiple handlers then randomly pick one
-                        else:
+                        elif len(valid_handlers) > 1:
                             handler = valid_handlers[
                                 randrange(len(valid_handlers))
                             ]
+                            handled = True
                             self.handle_event(handler, event)
+                            break
+
+                if not handled:
+                    print_debug(
+                        self._print_target, 
+                        self.debug_level, 
+                        "Could not determine handler for event.", 
+                        DEBUG_INFO
+                    )
 
     def run_handler_conductor_interaction(self)->None:
         """Function to be run in its own thread, to handle any inbound messages
@@ -156,6 +169,7 @@ class MeowRunner:
             if self._stop_han_con_pipe[0] in ready:
                 return
             else:
+                executed = False
                 for from_handler in self.from_handlers:
                     if from_handler in ready:
                         # Read job directory from the handler channel
@@ -188,13 +202,26 @@ class MeowRunner:
                         # If we've only one conductor, use that
                         if len(valid_conductors) == 1:
                             conductor = valid_conductors[0]
+                            executed = True
                             self.execute_job(conductor, job_dir)
+                            break
                         # If multiple handlers then randomly pick one
-                        else:
+                        elif len(valid_conductors) > 1:
                             conductor = valid_conductors[
                                 randrange(len(valid_conductors))
                             ]
+                            executed = True
                             self.execute_job(conductor, job_dir)
+                            break
+
+                # TODO determine something more useful to do here
+                if not executed:
+                    print_debug(
+                        self._print_target, 
+                        self.debug_level, 
+                        f"No conductor could be found for job {job_dir}", 
+                        DEBUG_INFO
+                    )
 
     def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None:
         """Function for a given handler to handle a given event, without 

functionality/debug.py

@@ -6,8 +6,8 @@ Author(s): David Marchant
 
 from typing import Any, Tuple
 
-from core.correctness.validation import check_type
+from meow_base.core.correctness.validation import check_type
-from core.correctness.vars import DEBUG_INFO, DEBUG_WARNING
+from meow_base.core.correctness.vars import DEBUG_INFO, DEBUG_WARNING
 
 
 def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:

functionality/file_io.py

@@ -11,7 +11,7 @@ from os import makedirs, remove, rmdir, walk
 from os.path import exists, isfile, join
 from typing import Any, Dict, List
 
-from core.correctness.validation import valid_path
+from meow_base.core.correctness.validation import valid_path
 
 
 def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):

functionality/hashing.py

@@ -6,8 +6,8 @@ Author(s): David Marchant
 
 from hashlib import sha256
 
-from core.correctness.vars import HASH_BUFFER_SIZE, SHA256
+from meow_base.core.correctness.vars import HASH_BUFFER_SIZE, SHA256
-from core.correctness.validation import check_type, valid_existing_file_path
+from meow_base.core.correctness.validation import check_type, valid_existing_file_path
 
 def _get_file_sha256(file_path):
     sha256_hash = sha256()

functionality/meow.py

@@ -8,16 +8,16 @@ from datetime import datetime
 from os.path import basename, dirname, relpath, splitext
 from typing import Any, Dict, Union, List
 
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.validation import check_type, valid_dict, valid_list
+from meow_base.core.correctness.validation import check_type, valid_dict, valid_list
-from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
+from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
     EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \
     JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \
     STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, SWEEP_START, \
     SWEEP_STOP
-from functionality.naming import generate_job_id, generate_rule_id
+from meow_base.functionality.naming import generate_job_id, generate_rule_id
 
 # mig trigger keyword replacements
 KEYWORD_PATH = "{PATH}"
@@ -208,7 +208,7 @@ def create_rule(pattern:BasePattern, recipe:BaseRecipe,
 
     # TODO fix me
     # Imported here to avoid circular imports at top of file
-    import rules
+    import meow_base.rules
     all_rules = {
         (r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()
     }

functionality/naming.py

@@ -7,7 +7,7 @@ Author(s): David Marchant
 from typing import List
 from random import SystemRandom
 
-from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
+from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
 
 
 #TODO Make this guaranteed unique

functionality/parameterisation.py

@@ -10,7 +10,7 @@ from os import getenv
 from papermill.translators import papermill_translators
 from typing import Any, Dict, List
 
-from core.correctness.validation import check_script, check_type
+from meow_base.core.correctness.validation import check_script, check_type
 
 # Adapted from: https://github.com/rasmunk/notebook_parameterizer
 def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any], 

functionality/process_io.py

@@ -12,7 +12,7 @@ from multiprocessing.connection import Connection, wait as multi_wait
 if osName == 'nt':
     from multiprocessing.connection import PipeConnection
 from multiprocessing.queues import Queue
-from core.correctness.vars import VALID_CHANNELS
+from meow_base.core.correctness.vars import VALID_CHANNELS
 
 
 def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:

functionality/requirements.py

@@ -11,7 +11,7 @@ from os.path import basename
 from sys import version_info, prefix, base_prefix
 from typing import Any, Dict, List, Tuple, Union
 
-from core.correctness.validation import check_type
+from meow_base.core.correctness.validation import check_type
 
 REQUIREMENT_PYTHON = "python"
 REQ_PYTHON_MODULES = "modules"

patterns/__init__.py

@@ -1,2 +1,2 @@
 
-from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor
+from .file_event_pattern import FileEventPattern, WatchdogMonitor

patterns/file_event_pattern.py

@@ -18,19 +18,19 @@ from typing import Any, Union, Dict, List
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_monitor import BaseMonitor
+from meow_base.core.base_monitor import BaseMonitor
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.validation import check_type, valid_string, \
+from meow_base.core.correctness.validation import check_type, valid_string, \
     valid_dict, valid_list, valid_path, valid_dir_path
-from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
+from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
     VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
     FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
     FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT
-from functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.debug import setup_debugging, print_debug
-from functionality.hashing import get_file_hash
+from meow_base.functionality.hashing import get_file_hash
-from functionality.meow import create_rule, create_watchdog_event
+from meow_base.functionality.meow import create_rule, create_watchdog_event
 
 # Events that are monitored by default
 _DEFAULT_MASK = [

recipes/__init__.py

@@ -1,4 +1,4 @@
 
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
+from .jupyter_notebook_recipe import JupyterNotebookRecipe, PapermillHandler, \
-    PapermillHandler
+    getRecipeFromNotebook
-from recipes.python_recipe import PythonRecipe, PythonHandler
+from .python_recipe import PythonRecipe, PythonHandler

recipes/jupyter_notebook_recipe.py

@@ -11,21 +11,21 @@ import sys
 
 from typing import Any, Tuple, Dict
 
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.correctness.meow import valid_event
+from meow_base.core.correctness.meow import valid_event
-from core.correctness.validation import check_type, valid_string, \
+from meow_base.core.correctness.validation import check_type, valid_string, \
-    valid_dict, valid_path, valid_dir_path
+    valid_dict, valid_path, valid_dir_path, valid_existing_file_path
-from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
+from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
     EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \
     JOB_ID, WATCHDOG_BASE, META_FILE, \
     PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
     EVENT_RULE, get_base_file
-from functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.debug import setup_debugging, print_debug
-from functionality.file_io import make_dir, read_notebook, write_notebook, \
+from meow_base.functionality.file_io import make_dir, read_notebook, write_notebook, \
     write_yaml
-from functionality.meow import create_job, replace_keywords
+from meow_base.functionality.meow import create_job, replace_keywords
 
 
 class JupyterNotebookRecipe(BaseRecipe):
@@ -182,20 +182,37 @@ class PapermillHandler(BaseHandler):
         # Send job directory, as actual definitons will be read from within it
         self.to_runner.send(job_dir)
 
+#TODO test me
+def getRecipeFromNotebook(name:str, notebook_filename:str, 
+        parameters:Dict[str,Any]={}, requirements:Dict[str,Any]={}
+        )->JupyterNotebookRecipe:
+    valid_existing_file_path(notebook_filename, extension=".ipynb")
+    check_type(name, str, hint="getRecipeFromNotebook.name")
+
+    notebook_code = read_notebook(notebook_filename)
+
+    return JupyterNotebookRecipe(
+        name, 
+        notebook_code, 
+        parameters=parameters, 
+        requirements=requirements, 
+        source=notebook_filename
+    )
+
 # Papermill job execution code, to be run within the conductor
 def papermill_job_func(job_dir):
     # Requires own imports as will be run in its own execution environment
     import os
     import papermill
     from datetime import datetime
-    from core.correctness.vars import JOB_EVENT, JOB_ID, \
+    from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
         EVENT_PATH, META_FILE, PARAMS_FILE, \
         JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
         JOB_ERROR, STATUS_FAILED, get_job_file, \
         get_result_file
-    from functionality.file_io import read_yaml, write_notebook, write_yaml
+    from meow_base.functionality.file_io import read_yaml, write_notebook, write_yaml
-    from functionality.hashing import get_file_hash
+    from meow_base.functionality.hashing import get_file_hash
-    from functionality.parameterisation import parameterize_jupyter_notebook
+    from meow_base.functionality.parameterisation import parameterize_jupyter_notebook
 
 
     # Identify job files

recipes/python_recipe.py

@@ -10,21 +10,21 @@ import sys
 
 from typing import Any, Tuple, Dict, List
 
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.correctness.meow import valid_event
+from meow_base.core.correctness.meow import valid_event
-from core.correctness.validation import check_script, valid_string, \
+from meow_base.core.correctness.validation import check_script, valid_string, \
     valid_dict, valid_dir_path
-from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
+from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
     DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
     EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
     JOB_ID, WATCHDOG_BASE, META_FILE, \
     PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
     get_base_file
-from functionality.debug import setup_debugging, print_debug
+from meow_base.functionality.debug import setup_debugging, print_debug
-from functionality.file_io import make_dir, read_file_lines, write_file, \
+from meow_base.functionality.file_io import make_dir, read_file_lines, write_file, \
     write_yaml, lines_to_string
-from functionality.meow import create_job, replace_keywords
+from meow_base.functionality.meow import create_job, replace_keywords
 
 
 class PythonRecipe(BaseRecipe):
@@ -180,14 +180,14 @@ def python_job_func(job_dir):
     import os
     from datetime import datetime
     from io import StringIO
-    from core.correctness.vars import JOB_EVENT, JOB_ID, \
+    from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
         EVENT_PATH, META_FILE, PARAMS_FILE, \
         JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
         JOB_ERROR, STATUS_FAILED, get_base_file, \
         get_job_file, get_result_file
-    from functionality.file_io import read_yaml, write_yaml
+    from meow_base.functionality.file_io import read_yaml, write_yaml
-    from functionality.hashing import get_file_hash
+    from meow_base.functionality.hashing import get_file_hash
-    from functionality.parameterisation import parameterize_python_script
+    from meow_base.functionality.parameterisation import parameterize_python_script
 
     # Identify job files
     meta_file = os.path.join(job_dir, META_FILE)

rules/__init__.py

@@ -1,3 +1,3 @@
 
-from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
+from .file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
-from rules.file_event_python_rule import FileEventPythonRule
+from .file_event_python_rule import FileEventPythonRule

rules/file_event_jupyter_notebook_rule.py

@@ -6,10 +6,10 @@ and JupyterNotebookRecipe.
 Author(s): David Marchant
 """
 
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.validation import check_type
+from meow_base.core.correctness.validation import check_type
-from patterns.file_event_pattern import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
 
 # TODO potentailly remove this and just invoke BaseRule directly, as does not
 # add any functionality other than some validation.

rules/file_event_python_rule.py

@@ -6,10 +6,10 @@ and PythonRecipe.
 Author(s): David Marchant
 """
 
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.validation import check_type
+from meow_base.core.correctness.validation import check_type
-from patterns.file_event_pattern import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from recipes.python_recipe import PythonRecipe
+from meow_base.recipes.python_recipe import PythonRecipe
 
 # TODO potentailly remove this and just invoke BaseRule directly, as does not
 # add any functionality other than some validation.

tests/shared.py

@@ -7,10 +7,10 @@ import os
 
 from distutils.dir_util import copy_tree
 
-from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
+from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
-from functionality.file_io import make_dir, rmtree
+from meow_base.functionality.file_io import make_dir, rmtree
-from patterns import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from recipes import JupyterNotebookRecipe
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
 
 # testing 
 TEST_DIR = "test_files"

tests/test_base.py

@@ -3,14 +3,14 @@ import unittest
  
 from typing import Any, Union, Tuple, Dict
 
-from core.base_conductor import BaseConductor
+from meow_base.core.base_conductor import BaseConductor
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.base_monitor import BaseMonitor
+from meow_base.core.base_monitor import BaseMonitor
-from core.base_pattern import BasePattern
+from meow_base.core.base_pattern import BasePattern
-from core.base_recipe import BaseRecipe
+from meow_base.core.base_recipe import BaseRecipe
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
+from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
-from patterns import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
 from shared import setup, teardown, valid_pattern_one, valid_recipe_one
 
 

tests/test_conductors.py

@@ -5,21 +5,21 @@ import unittest
 from datetime import datetime
 from typing import Dict
 
-from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
+from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
     JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, JOB_EVENT, \
     META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, JOB_PATTERN, \
     STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
     JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
     EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
-from conductors import LocalPythonConductor
+from meow_base.conductors import LocalPythonConductor
-from functionality.file_io import read_file, read_yaml, write_file, \
+from meow_base.functionality.file_io import read_file, read_yaml, write_file, \
     write_notebook, write_yaml, lines_to_string, make_dir
-from functionality.hashing import get_file_hash
+from meow_base.functionality.hashing import get_file_hash
-from functionality.meow import create_watchdog_event, create_job, create_rule
+from meow_base.functionality.meow import create_watchdog_event, create_job, create_rule
-from patterns import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
     papermill_job_func
-from recipes.python_recipe import PythonRecipe, python_job_func
+from meow_base.recipes.python_recipe import PythonRecipe, python_job_func
 from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
     TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, \
     BAREBONES_PYTHON_SCRIPT, BAREBONES_NOTEBOOK

tests/test_functionality.py

@@ -11,34 +11,34 @@ from sys import prefix, base_prefix
 from time import sleep
 from typing import Dict
 
-from core.base_rule import BaseRule
+from meow_base.core.base_rule import BaseRule
-from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
+from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
     SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
     WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
     PYTHON_FUNC, JOB_ID, JOB_EVENT, \
     JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
     JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
-from functionality.debug import setup_debugging
+from meow_base.functionality.debug import setup_debugging
-from functionality.file_io import lines_to_string, make_dir, read_file, \
+from meow_base.functionality.file_io import lines_to_string, make_dir, read_file, \
     read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
     write_notebook, write_yaml    
-from functionality.hashing import get_file_hash
+from meow_base.functionality.hashing import get_file_hash
-from functionality.meow import create_event, create_job, create_rule, \
+from meow_base.functionality.meow import create_event, create_job, create_rule, \
     create_rules, create_watchdog_event, replace_keywords, \
     create_parameter_sweep, \
     KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
     KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
     KEYWORD_REL_PATH
-from functionality.naming import _generate_id
+from meow_base.functionality.naming import _generate_id
-from functionality.parameterisation import parameterize_jupyter_notebook, \
+from meow_base.functionality.parameterisation import parameterize_jupyter_notebook, \
     parameterize_python_script
-from functionality.process_io import wait
+from meow_base.functionality.process_io import wait
-from functionality.requirements import create_python_requirements, \
+from meow_base.functionality.requirements import create_python_requirements, \
     check_requirements, \
     REQUIREMENT_PYTHON, REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, \
     REQ_PYTHON_VERSION
-from patterns import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from recipes import JupyterNotebookRecipe
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
 from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
     valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
     COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
@@ -594,7 +594,7 @@ class MeowTests(unittest.TestCase):
 
         self.assertEqual(len(rules), 0)
 
-    # Test that create_rules creates rules from patterns and recipes
+    # Test that create_rules creates rules from meow_base.patterns and recipes
     def testCreateRulesPatternsAndRecipesDicts(self)->None:
         patterns = {
             valid_pattern_one.name: valid_pattern_one,

tests/test_patterns.py

@@ -5,13 +5,13 @@ import unittest
 
 from multiprocessing import Pipe
 
-from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
+from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
     EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
     SWEEP_JUMP, SWEEP_STOP
-from functionality.file_io import make_dir
+from meow_base.functionality.file_io import make_dir
-from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
+from meow_base.patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
     _DEFAULT_MASK
-from recipes import JupyterNotebookRecipe
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
 from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE
 
 

tests/test_recipes.py

@@ -6,23 +6,23 @@ import unittest
 from multiprocessing import Pipe
 from typing import Dict
 
-from core.correctness.meow import valid_job
+from meow_base.core.correctness.meow import valid_job
-from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
+from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
     EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
     JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
     META_FILE, JOB_ERROR, \
     PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
     get_base_file, get_job_file, get_result_file
-from functionality.file_io import lines_to_string, make_dir, read_yaml, \
+from meow_base.functionality.file_io import lines_to_string, make_dir, read_yaml, \
     write_file, write_notebook, write_yaml
-from functionality.hashing import get_file_hash
+from meow_base.functionality.hashing import get_file_hash
-from functionality.meow import create_job, create_rules, create_rule, \
+from meow_base.functionality.meow import create_job, create_rules, create_rule, \
     create_watchdog_event
-from patterns.file_event_pattern import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
     PapermillHandler, papermill_job_func
-from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
+from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
-from rules import FileEventJupyterNotebookRule, FileEventPythonRule
+from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule
 from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
     COMPLETE_PYTHON_SCRIPT, TEST_JOB_QUEUE, TEST_MONITOR_BASE, \
     TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK

tests/test_rules.py

@@ -1,9 +1,9 @@
 
 import unittest
 
-from patterns.file_event_pattern import FileEventPattern
+from meow_base.patterns.file_event_pattern import FileEventPattern
-from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
+from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
-from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
+from meow_base.rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
 from shared import setup, teardown, BAREBONES_NOTEBOOK
 
 class CorrectnessTests(unittest.TestCase):

tests/test_runner.py

@@ -8,21 +8,21 @@ from random import shuffle
 from shutil import copy
 from time import sleep
 
-from core.base_conductor import BaseConductor
+from meow_base.core.base_conductor import BaseConductor
-from core.base_handler import BaseHandler
+from meow_base.core.base_handler import BaseHandler
-from core.base_monitor import BaseMonitor
+from meow_base.core.base_monitor import BaseMonitor
-from conductors import LocalPythonConductor
+from meow_base.conductors import LocalPythonConductor
-from core.correctness.vars import get_result_file, \
+from meow_base.core.correctness.vars import get_result_file, \
     JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME
-from core.runner import MeowRunner
+from meow_base.core.runner import MeowRunner
-from functionality.file_io import make_dir, read_file, read_notebook, \
+from meow_base.functionality.file_io import make_dir, read_file, read_notebook, \
     read_yaml, write_file, lines_to_string
-from functionality.meow import create_parameter_sweep
+from meow_base.functionality.meow import create_parameter_sweep
-from functionality.requirements import create_python_requirements
+from meow_base.functionality.requirements import create_python_requirements
-from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
+from meow_base.patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
-from recipes.jupyter_notebook_recipe import PapermillHandler, \
+from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler, \
     JupyterNotebookRecipe
-from recipes.python_recipe import PythonHandler, PythonRecipe
+from meow_base.recipes.python_recipe import PythonHandler, PythonRecipe
 from shared import setup, teardown, backup_before_teardown, \
     TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, MAKER_RECIPE, \
     APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, FILTER_RECIPE, \
@@ -916,10 +916,6 @@ class MeowTests(unittest.TestCase):
         output = read_file(os.path.join(output_path))
         self.assertEqual(output, "12505000.0")
 
-
-
-
-
     def testSelfModifyingAnalysis(self)->None:
         maker_pattern = FileEventPattern(
             "maker_pattern", 

tests/test_validation.py

@@ -5,16 +5,16 @@ import os
 from datetime import datetime
 from typing import Any, Union
 
-from core.correctness.meow import valid_event, valid_job, valid_watchdog_event
+from meow_base.core.correctness.meow import valid_event, valid_job, valid_watchdog_event
-from core.correctness.validation import check_type, check_implementation, \
+from meow_base.core.correctness.validation import check_type, check_implementation, \
     valid_string, valid_dict, valid_list, valid_existing_file_path, \
     valid_dir_path, valid_non_existing_path, check_callable
-from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
+from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
     EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
     JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
     WATCHDOG_HASH
-from functionality.file_io import make_dir
+from meow_base.functionality.file_io import make_dir
-from functionality.meow import create_rule
+from meow_base.functionality.meow import create_rule
 from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \
     valid_recipe_one