NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformatted imports to work better on other machines, plus added benchmarking to project

Browse Source
This commit is contained in:
PatchOfScotland
2023-03-13 11:32:45 +01:00
parent c01df1b190
commit 40ed98000b
48 changed files with 907 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -58,6 +58,10 @@ tests/test_data
tests/job_output
tests/job_queue
tests/Backup*
benchmarking/benchmark_base
benchmarking/job_output
benchmarking/job_queue
benchmarking/result*
# hdf5
*.h5

34
benchmarking/mrme.py Normal file
View File

@ -0,0 +1,34 @@
from meow_base.patterns import FileEventPattern
from meow_base.recipes import getRecipeFromNotebook
from shared import run_test, MRME
def multiple_rules_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
patterns = {}
for i in range(job_count):
pattern = FileEventPattern(
f"pattern_{i}",
f"testing/file_{i}.txt",
"recipe_one",
"input"
)
patterns[pattern.name] = pattern
recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
recipes = {
recipe.name: recipe
}
run_test(
patterns,
recipes,
job_count,
job_count,
REPEATS,
job_counter,
requested_jobs,
runtime_start,
signature=MRME
)

34
benchmarking/mrse.py Normal file
View File

@ -0,0 +1,34 @@
from meow_base.patterns import FileEventPattern
from meow_base.recipes import getRecipeFromNotebook
from shared import run_test, MRSE
def multiple_rules_single_event(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
patterns = {}
for i in range(job_count):
pattern = FileEventPattern(
f"pattern_{i}",
f"testing/*",
"recipe_one",
"input"
)
patterns[pattern.name] = pattern
recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
recipes = {
recipe.name: recipe
}
run_test(
patterns,
recipes,
1,
job_count,
REPEATS,
job_counter,
requested_jobs,
runtime_start,
signature=MRSE
)

175
benchmarking/run_all.py Normal file
View File

@ -0,0 +1,175 @@
import matplotlib.pyplot as pyplot
import numpy
import sys
import time
import os
from shared import JOBS_COUNTS, REPEATS, TESTS, MRME, MRSE, SRME, SRSEP, SRSES, RESULTS_DIR, BASE, GRAPH_FILENAME
from mrme import multiple_rules_multiple_events
from mrse import multiple_rules_single_event
from srme import single_rule_multiple_events
from srsep import single_rule_single_event_parallel
from srsps import single_rule_single_event_sequential
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from meow_base.functionality.file_io import rmtree
LINE_KEYS = {
SRSES: ('x','#a1467e'),
SRME: ('.','#896cff'),
MRME: ('d','#5983b0'),
MRSE: ('P','#ff6cbe'),
SRSEP: ('*','#3faf46'),
}
def run_tests():
rmtree(RESULTS_DIR)
requested_jobs=0
for job_count in JOBS_COUNTS:
requested_jobs += job_count * REPEATS * len(TESTS)
print(f"requested_jobs: {requested_jobs}")
runtime_start=time.time()
job_counter=0
for job_count in JOBS_COUNTS:
for test in TESTS:
if test == MRME:
multiple_rules_multiple_events(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
job_counter += job_count * REPEATS
elif test == MRSE:
multiple_rules_single_event(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
job_counter += job_count * REPEATS
elif test == SRME:
single_rule_multiple_events(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
job_counter += job_count * REPEATS
elif test == SRSEP:
single_rule_single_event_parallel(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
job_counter += job_count * REPEATS
elif test == SRSES:
single_rule_single_event_sequential(job_count, REPEATS, job_counter, requested_jobs, runtime_start)
job_counter += job_count * REPEATS
print(f"All tests completed in: {str(time.time()-runtime_start)}")
def get_meow_graph(results_dir):
lines = []
for run_type in os.listdir(results_dir):
#if run_type == 'single_Pattern_single_file_sequential':
# continue
# lines.append((f'scheduling {run_type}', [], 'solid'))
lines.append((run_type, [], 'solid'))
run_type_path = os.path.join(results_dir, run_type)
for job_count in os.listdir(run_type_path):
results_path = os.path.join(run_type_path, job_count, 'results.txt')
with open(results_path, 'r') as f_in:
data = f_in.readlines()
scheduling_duration = 0
for line in data:
if "Average schedule time: " in line:
scheduling_duration = float(line.replace("Average schedule time: ", ''))
lines[-1][1].append((job_count, scheduling_duration))
lines[-1][1].sort(key=lambda y: float(y[0]))
return lines
def make_plot(lines, graph_path, title, logged):
w = 10
h = 4
linecount = 0
columns = 1
pyplot.figure(figsize=(w, h))
for l in range(len(lines)):
x_values = numpy.asarray([float(i[0]) for i in lines[l][1]])
y_values = numpy.asarray([float(i[1]) for i in lines[l][1]])
# Remove this check to always display lines
if lines[l][2] == 'solid':
pyplot.plot(x_values, y_values, label=lines[l][0], linestyle=lines[l][2], marker=LINE_KEYS[lines[l][0]][0], color=LINE_KEYS[lines[l][0]][1])
linecount += 1
columns = int(linecount/3) + 1
pyplot.xlabel("Number of jobs scheduled")
pyplot.ylabel("Time taken (seconds)")
pyplot.title(title)
handles, labels = pyplot.gca().get_legend_handles_labels()
# legend_order = [2, 4, 0, 1, 3]
# pyplot.legend([handles[i] for i in legend_order], [labels[i] for i in legend_order])
pyplot.legend(ncol=columns, prop={'size': 12})
if logged:
pyplot.yscale('log')
x_ticks = []
for tick in x_values:
label = int(tick)
if tick <= 100 and tick % 20 == 0:
label = f"\n{int(tick)}"
x_ticks.append(label)
pyplot.xticks(x_values, x_ticks)
pyplot.savefig(graph_path, format='pdf', bbox_inches='tight')
def make_both_plots(lines, path, title, log=True):
make_plot(lines, path, title, False)
if log:
logged_path = path[:path.index(".pdf")] + "_logged" + path[path.index(".pdf"):]
make_plot(lines, logged_path, title, True)
def make_graphs():
lines = get_meow_graph(RESULTS_DIR)
make_both_plots(lines, "result.pdf", "MiG scheduling overheads on the Threadripper")
average_lines = []
all_delta_lines = []
no_spsfs_delta_lines = []
for line_signature, line_values, lines_style in lines:
if lines_style == 'solid':
averages = [(i, v/float(i)) for i, v in line_values]
average_lines.append((line_signature, averages, lines_style))
if line_signature not in ["total single_Pattern_single_file_sequential", "scheduling single_Pattern_single_file_sequential_jobs", "SPSFS"]:
deltas = []
for i in range(len(line_values)-1):
deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) )
no_spsfs_delta_lines.append((line_signature, deltas, lines_style))
deltas = []
for i in range(len(line_values)-1):
deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) )
all_delta_lines.append((line_signature, deltas, lines_style))
make_both_plots(average_lines, "result_averaged.pdf", "Per-job MiG scheduling overheads on the Threadripper")
make_both_plots(all_delta_lines, "result_deltas.pdf", "Difference in per-job MiG scheduling overheads on the Threadripper", log=False)
if __name__ == '__main__':
try:
run_tests()
make_graphs()
rmtree(DEFAULT_JOB_QUEUE_DIR)
rmtree(DEFAULT_JOB_OUTPUT_DIR)
rmtree(BASE)
except KeyboardInterrupt as ki:
try:
sys.exit(1)
except SystemExit:
os._exit(1)

69
benchmarking/sequential.ipynb Normal file
View File

@ -0,0 +1,69 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"INPUT_FILE = 'file_0.txt'\n",
"MAX_COUNT = 100"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"with open(INPUT_FILE, 'r') as f:\n",
" data = int(f.read())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"read in: \"+ str(data))\n",
"print(\"writing out: \"+ str(data+1))"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"if data+1 < MAX_COUNT:\n",
" with open(INPUT_FILE.replace(str(data), str(data+1)), 'w') as f:\n",
" f.write(str(data+1))"
]
}
],
"metadata": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

225
benchmarking/shared.py Normal file
View File

@ -0,0 +1,225 @@
import datetime
import io
import os
import pathlib
import time
import yaml
from typing import Any, Dict, Tuple
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from meow_base.core.runner import MeowRunner
from meow_base.patterns.file_event_pattern import WatchdogMonitor
from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler
from meow_base.conductors import LocalPythonConductor
from meow_base.functionality.file_io import rmtree
RESULTS_DIR = "results"
BASE = "benchmark_base"
GRAPH_FILENAME = "graph.pdf"
REPEATS = 10
JOBS_COUNTS = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, 250, 300, 400, 500]
SRME = "single_rule_multiple_events"
MRSE = "multiple_rules_single_event"
SRSEP = "single_rule_single_event_parallel"
MRME = "multiple_rules_multiple_events"
SRSES = "single_rule_single_event_sequential"
TESTS = [
SRME,
MRSE,
SRSEP,
MRME,
# This test will take approx 90% of total time
SRSES
]
class DummyConductor(LocalPythonConductor):
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
return False, ">:("
def datetime_to_timestamp(date_time_obj):
return time.mktime(date_time_obj.timetuple()) + float(date_time_obj.microsecond)/1000000
def generate(file_count, file_path, file_type='.txt'):
first_filename = ''
start = time.time()
for i in range(int(file_count)):
filename = file_path + str(i) + file_type
if not first_filename:
first_filename = filename
with open(filename, 'w') as f:
f.write('0')
return first_filename, time.time() - start
def cleanup(jobs, file_out, base_time, gen_time, execution=False):
if not jobs:
return
job_timestamps = []
for job in jobs:
if execution:
with open(f"{DEFAULT_JOB_OUTPUT_DIR}/{job}/job.yml", 'r') as f_in:
data = yaml.load(f_in, Loader=yaml.Loader)
else:
with open(f"{DEFAULT_JOB_QUEUE_DIR}/{job}/job.yml", 'r') as f_in:
data = yaml.load(f_in, Loader=yaml.Loader)
create_datetime = data['create']
create_timestamp = datetime_to_timestamp(create_datetime)
job_timestamps.append((create_timestamp, create_datetime))
job_timestamps.sort(key=lambda y: int(y[0]))
first = job_timestamps[0]
last = job_timestamps[-1]
#dt = datetime.datetime.fromtimestamp(os.path.getctime(base_time), datetime.timezone(datetime.timedelta(hours=0)))
dt = datetime.datetime.fromtimestamp(os.path.getctime(base_time))
# if execution:
# queue_times = []
# execution_times = []
# for j in jobs:
# mrsl_dict = load(os.path.join(mrsl_dir, j))#
#
# queue_times.append(time.mktime(mrsl_dict['EXECUTING_TIMESTAMP']) - time.mktime(mrsl_dict['QUEUED_TIMESTAMP']))
# execution_times.append(time.mktime(mrsl_dict['FINISHED_TIMESTAMP']) - time.mktime(mrsl_dict['EXECUTING_TIMESTAMP']))
pathlib.Path(os.path.dirname(file_out)).mkdir(parents=True, exist_ok=True)
with open(file_out, 'w') as f_out:
f_out.write("Job count: "+ str(len(jobs)) +"\n")
f_out.write("Generation time: "+ str(round(gen_time, 5)) +"\n")
f_out.write("First trigger: "+ str(dt) +"\n")
f_out.write("First scheduling datetime: "+ str(first[1]) +"\n")
f_out.write("Last scheduling datetime: "+ str(last[1]) +"\n")
f_out.write("First scheduling unixtime: "+ str(first[0]) +"\n")
f_out.write("First scheduling unixtime: "+ str(last[0]) +"\n")
f_out.write("Scheduling difference (seconds): "+ str(round(last[0] - first[0], 3)) +"\n")
f_out.write("Initial scheduling delay (seconds): "+ str(round(first[0] - os.path.getctime(base_time), 3)) +"\n")
total_time = round(last[0] - os.path.getctime(base_time), 3)
f_out.write("Total scheduling delay (seconds): "+ str(total_time) +"\n")
# if execution:
# f_out.write("Average execution time (seconds): "+ str(round(mean(execution_times), 3)) +"\n")
# f_out.write("Max execution time (seconds): "+ str(round(max(execution_times), 3)) +"\n")
# f_out.write("Min execution time (seconds): "+ str(round(min(execution_times), 3)) +"\n")
# f_out.write("Average queueing delay (seconds): "+ str(round(mean(queue_times), 3)) +"\n")
# f_out.write("Max queueing delay (seconds): "+ str(round(max(queue_times), 3)) +"\n")
# f_out.write("Min queueing delay (seconds): "+ str(round(min(queue_times), 3)) +"\n")
# queue_times.remove(max(queue_times))
# f_out.write("Average excluded queueing delay (seconds): "+ str(round(mean(queue_times), 3)) +"\n")
return total_time
def mean(l):
return sum(l)/len(l)
def collate_results(base_results_dir):
scheduling_delays = []
for run in os.listdir(base_results_dir):
if run != 'results.txt':
with open(os.path.join(base_results_dir, run, 'results.txt'), 'r') as f:
d = f.readlines()
for l in d:
if "Total scheduling delay (seconds): " in l:
scheduling_delays.append(float(l.replace("Total scheduling delay (seconds): ", '')))
with open(os.path.join(base_results_dir, 'results.txt'), 'w') as f:
f.write(f"Average schedule time: {round(mean(scheduling_delays), 3)}\n")
f.write(f"Scheduling times: {scheduling_delays}")
def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_counter, requested_jobs, runtime_start, signature='', execution=False, print_logging=False):
if not os.path.exists(RESULTS_DIR):
os.mkdir(RESULTS_DIR)
# Does not work. left here as reminder
if execution:
os.system("export LC_ALL=C.UTF-8")
os.system("export LANG=C.UTF-8")
for run in range(repeats):
# Ensure complete cleanup from previous run
for f in [BASE, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR]:
if os.path.exists(f):
rmtree(f)
file_base = os.path.join(BASE, 'testing')
pathlib.Path(file_base).mkdir(parents=True, exist_ok=True)
runner_debug_stream = io.StringIO("")
if execution:
runner = MeowRunner(
WatchdogMonitor(BASE, patterns, recipes, settletime=1),
PapermillHandler(),
LocalPythonConductor(),
print=runner_debug_stream,
logging=3
)
else:
runner = MeowRunner(
WatchdogMonitor(BASE, patterns, recipes, settletime=1),
PapermillHandler(),
DummyConductor(),
print=runner_debug_stream,
logging=3
)
# meow.WorkflowRunner(
# VGRID,
# num_workers,
# patterns=patterns,
# recipes=recipes,
# daemon=True,
# start_workers=False,
# retro_active_jobs=False,
# print_logging=print_logging,
# file_logging=False,
# wait_time=1
# )
runner.start()
# Generate triggering files
first_filename, generation_duration = generate(files_count, file_base +"/file_")
idle_loops = 0
total_loops = 0
messages = 0
total_time = expected_job_count * 3
if execution:
total_time = expected_job_count * 5
while idle_loops < 10 and total_loops < total_time:
time.sleep(1)
runner_debug_stream.seek(0)
new_messages = len(runner_debug_stream.readlines())
if messages == new_messages:
idle_loops += 1
else:
idle_loops = 0
messages = new_messages
total_loops += 1
runner.stop()
if execution:
jobs = os.listdir(DEFAULT_JOB_OUTPUT_DIR)
else:
jobs = os.listdir(DEFAULT_JOB_QUEUE_DIR)
results_path = os.path.join(RESULTS_DIR, signature, str(expected_job_count), str(run), 'results.txt')
cleanup(jobs, results_path, first_filename, generation_duration, execution=execution)
print(f"Completed scheduling run {str(run + 1)} of {str(len(jobs))}/{str(expected_job_count)} jobs for '{signature}' {job_counter + expected_job_count*(run+1)}/{requested_jobs} ({str(round(time.time()-runtime_start, 3))}s)")
collate_results(os.path.join(RESULTS_DIR, signature, str(expected_job_count)))

33
benchmarking/srme.py Normal file
View File

@ -0,0 +1,33 @@
from meow_base.patterns import FileEventPattern
from meow_base.recipes import getRecipeFromNotebook
from shared import run_test, SRME
def single_rule_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
patterns = {}
pattern = FileEventPattern(
f"pattern_one",
f"testing/*",
"recipe_one",
"input"
)
patterns[pattern.name] = pattern
recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
recipes = {
recipe.name: recipe
}
run_test(
patterns,
recipes,
job_count,
job_count,
REPEATS,
job_counter,
requested_jobs,
runtime_start,
signature=SRME
)

35
benchmarking/srsep.py Normal file
View File

@ -0,0 +1,35 @@
from meow_base.patterns import FileEventPattern
from meow_base.recipes import getRecipeFromNotebook
from meow_base.functionality.meow import create_parameter_sweep
from shared import run_test, SRSEP
def single_rule_single_event_parallel(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
patterns = {}
pattern = FileEventPattern(
f"pattern_one",
f"testing/*",
"recipe_one",
"input",
sweep=create_parameter_sweep("var", 1, job_count, 1)
)
patterns[pattern.name] = pattern
recipe = getRecipeFromNotebook("recipe_one", "test.ipynb")
recipes = {
recipe.name: recipe
}
run_test(
patterns,
recipes,
1,
job_count,
REPEATS,
job_counter,
requested_jobs,
runtime_start,
signature=SRSEP
)

38
benchmarking/srsps.py Normal file
View File

@ -0,0 +1,38 @@
from meow_base.patterns import FileEventPattern
from meow_base.recipes import getRecipeFromNotebook
from shared import run_test, SRSES
def single_rule_single_event_sequential(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
patterns = {}
pattern = FileEventPattern(
f"pattern_one",
f"testing/*",
"recipe_two",
"INPUT_FILE",
parameters={
"MAX_COUNT":job_count
}
)
patterns[pattern.name] = pattern
recipe = getRecipeFromNotebook("recipe_two", "sequential.ipynb")
recipes = {
recipe.name: recipe
}
run_test(
patterns,
recipes,
1,
job_count,
REPEATS,
job_counter,
requested_jobs,
runtime_start,
signature=SRSES,
execution=True,
print_logging=False
)

47
benchmarking/test.ipynb Normal file
View File

@ -0,0 +1,47 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"ename": "",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[1;31mFailed to start the Kernel. \n",
"Failed to start the Kernel 'Python 3.6.9 64-bit'. \n",
"View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details. Kernel has not been started"
]
}
],
"source": [
"print('this is some outpug ')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"interpreter": {
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
},
"kernelspec": {
"display_name": "Python 3.6.9 64-bit",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.6.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

2
conductors/__init__.py
View File

@ -1,2 +1,2 @@
from conductors.local_python_conductor import LocalPythonConductor
from .local_python_conductor import LocalPythonConductor

10
conductors/local_python_conductor.py
View File

@ -11,14 +11,14 @@ import shutil
from datetime import datetime
from typing import Any, Tuple, Dict
from core.base_conductor import BaseConductor
from core.correctness.meow import valid_job
from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
from core.correctness.validation import valid_dir_path
from functionality.file_io import make_dir, read_yaml, write_file, write_yaml
from meow_base.core.correctness.validation import valid_dir_path
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, write_yaml
class LocalPythonConductor(BaseConductor):
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR,

0
core/__init__.py Normal file
View File

4
core/base_conductor.py
View File

@ -8,8 +8,8 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from core.correctness.vars import get_drt_imp_msg
from core.correctness.validation import check_implementation
from meow_base.core.correctness.vars import get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation
class BaseConductor:

4
core/base_handler.py
View File

@ -8,8 +8,8 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from core.correctness.validation import check_implementation
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from meow_base.core.correctness.validation import check_implementation
class BaseHandler:

12
core/base_monitor.py
View File

@ -9,12 +9,12 @@ Author(s): David Marchant
from copy import deepcopy
from typing import Union, Dict
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from core.correctness.validation import check_implementation
from functionality.meow import create_rules
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from meow_base.core.correctness.validation import check_implementation
from meow_base.functionality.meow import create_rules
class BaseMonitor:

4
core/base_pattern.py
View File

@ -10,9 +10,9 @@ import itertools
from typing import Any, Union, Tuple, Dict, List
from core.correctness.vars import get_drt_imp_msg, \
from meow_base.core.correctness.vars import get_drt_imp_msg, \
VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
from core.correctness.validation import valid_string, check_type, \
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation, valid_dict

4
core/base_recipe.py
View File

@ -8,8 +8,8 @@ Author(s): David Marchant
from typing import Any, Dict
from core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
from core.correctness.validation import valid_string, check_implementation
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
from meow_base.core.correctness.validation import valid_string, check_implementation
class BaseRecipe:

8
core/base_rule.py
View File

@ -10,11 +10,11 @@ from sys import modules
from typing import Any
if "BasePattern" not in modules:
from core.base_pattern import BasePattern
from meow_base.core.base_pattern import BasePattern
if "BaseRecipe" not in modules:
from core.base_recipe import BaseRecipe
from core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
from core.correctness.validation import valid_string, check_type, \
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation

6
core/correctness/meow.py
View File

@ -2,9 +2,9 @@
from datetime import datetime
from typing import Any, Dict, Type
from core.base_rule import BaseRule
from core.correctness.validation import check_type
from core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type
from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \
JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH

2
core/correctness/validation.py
View File

@ -11,7 +11,7 @@ from os.path import sep, exists, isfile, isdir, dirname
from typing import Any, _SpecialForm, Union, Type, Dict, List, \
get_origin, get_args
from core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
from meow_base.core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
or_none:bool=False, hint:str="")->None:

49
core/runner.py
View File

@ -14,16 +14,16 @@ from multiprocessing import Pipe
from random import randrange
from typing import Any, Union, Dict, List
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR, \
EVENT_PATH
from core.correctness.validation import check_type, valid_list, valid_dir_path
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_yaml
from functionality.process_io import wait
from meow_base.core.correctness.validation import check_type, valid_list, valid_dir_path
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_yaml
from meow_base.functionality.process_io import wait
class MeowRunner:
@ -112,6 +112,7 @@ class MeowRunner:
if self._stop_mon_han_pipe[0] in ready:
return
else:
handled = False
for from_monitor in self.from_monitors:
if from_monitor in ready:
# Read event from the monitor channel
@ -136,13 +137,25 @@ class MeowRunner:
# If we've only one handler, use that
if len(valid_handlers) == 1:
handler = valid_handlers[0]
handled = True
self.handle_event(handler, event)
break
# If multiple handlers then randomly pick one
else:
elif len(valid_handlers) > 1:
handler = valid_handlers[
randrange(len(valid_handlers))
]
handled = True
self.handle_event(handler, event)
break
if not handled:
print_debug(
self._print_target,
self.debug_level,
"Could not determine handler for event.",
DEBUG_INFO
)
def run_handler_conductor_interaction(self)->None:
"""Function to be run in its own thread, to handle any inbound messages
@ -156,6 +169,7 @@ class MeowRunner:
if self._stop_han_con_pipe[0] in ready:
return
else:
executed = False
for from_handler in self.from_handlers:
if from_handler in ready:
# Read job directory from the handler channel
@ -188,14 +202,27 @@ class MeowRunner:
# If we've only one conductor, use that
if len(valid_conductors) == 1:
conductor = valid_conductors[0]
executed = True
self.execute_job(conductor, job_dir)
break
# If multiple handlers then randomly pick one
else:
elif len(valid_conductors) > 1:
conductor = valid_conductors[
randrange(len(valid_conductors))
]
executed = True
self.execute_job(conductor, job_dir)
break
# TODO determine something more useful to do here
if not executed:
print_debug(
self._print_target,
self.debug_level,
f"No conductor could be found for job {job_dir}",
DEBUG_INFO
)
def handle_event(self, handler:BaseHandler, event:Dict[str,Any])->None:
"""Function for a given handler to handle a given event, without
crashing the runner in the event of a problem."""

0
functionality/__init__.py Normal file
View File

4
functionality/debug.py
View File

@ -6,8 +6,8 @@ Author(s): David Marchant
from typing import Any, Tuple
from core.correctness.validation import check_type
from core.correctness.vars import DEBUG_INFO, DEBUG_WARNING
from meow_base.core.correctness.validation import check_type
from meow_base.core.correctness.vars import DEBUG_INFO, DEBUG_WARNING
def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:

2
functionality/file_io.py
View File

@ -11,7 +11,7 @@ from os import makedirs, remove, rmdir, walk
from os.path import exists, isfile, join
from typing import Any, Dict, List
from core.correctness.validation import valid_path
from meow_base.core.correctness.validation import valid_path
def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):

4
functionality/hashing.py
View File

@ -6,8 +6,8 @@ Author(s): David Marchant
from hashlib import sha256
from core.correctness.vars import HASH_BUFFER_SIZE, SHA256
from core.correctness.validation import check_type, valid_existing_file_path
from meow_base.core.correctness.vars import HASH_BUFFER_SIZE, SHA256
from meow_base.core.correctness.validation import check_type, valid_existing_file_path
def _get_file_sha256(file_path):
sha256_hash = sha256()

14
functionality/meow.py
View File

@ -8,16 +8,16 @@ from datetime import datetime
from os.path import basename, dirname, relpath, splitext
from typing import Any, Dict, Union, List
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.validation import check_type, valid_dict, valid_list
from core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type, valid_dict, valid_list
from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \
STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, SWEEP_START, \
SWEEP_STOP
from functionality.naming import generate_job_id, generate_rule_id
from meow_base.functionality.naming import generate_job_id, generate_rule_id
# mig trigger keyword replacements
KEYWORD_PATH = "{PATH}"
@ -208,7 +208,7 @@ def create_rule(pattern:BasePattern, recipe:BaseRecipe,
# TODO fix me
# Imported here to avoid circular imports at top of file
import rules
import meow_base.rules
all_rules = {
(r.pattern_type, r.recipe_type):r for r in BaseRule.__subclasses__()
}

2
functionality/naming.py
View File

@ -7,7 +7,7 @@ Author(s): David Marchant
from typing import List
from random import SystemRandom
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
#TODO Make this guaranteed unique

2
functionality/parameterisation.py
View File

@ -10,7 +10,7 @@ from os import getenv
from papermill.translators import papermill_translators
from typing import Any, Dict, List
from core.correctness.validation import check_script, check_type
from meow_base.core.correctness.validation import check_script, check_type
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any],

2
functionality/process_io.py
View File

@ -12,7 +12,7 @@ from multiprocessing.connection import Connection, wait as multi_wait
if osName == 'nt':
from multiprocessing.connection import PipeConnection
from multiprocessing.queues import Queue
from core.correctness.vars import VALID_CHANNELS
from meow_base.core.correctness.vars import VALID_CHANNELS
def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:

2
functionality/requirements.py
View File

@ -11,7 +11,7 @@ from os.path import basename
from sys import version_info, prefix, base_prefix
from typing import Any, Dict, List, Tuple, Union
from core.correctness.validation import check_type
from meow_base.core.correctness.validation import check_type
REQUIREMENT_PYTHON = "python"
REQ_PYTHON_MODULES = "modules"

2
patterns/__init__.py
View File

@ -1,2 +1,2 @@
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor
from .file_event_pattern import FileEventPattern, WatchdogMonitor

18
patterns/file_event_pattern.py
View File

@ -18,19 +18,19 @@ from typing import Any, Union, Dict, List
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from core.base_recipe import BaseRecipe
from core.base_monitor import BaseMonitor
from core.base_pattern import BasePattern
from core.base_rule import BaseRule
from core.correctness.validation import check_type, valid_string, \
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type, valid_string, \
valid_dict, valid_list, valid_path, valid_dir_path
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT
from functionality.debug import setup_debugging, print_debug
from functionality.hashing import get_file_hash
from functionality.meow import create_rule, create_watchdog_event
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_rule, create_watchdog_event
# Events that are monitored by default
_DEFAULT_MASK = [

6
recipes/__init__.py
View File

@ -1,4 +1,4 @@
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler
from recipes.python_recipe import PythonRecipe, PythonHandler
from .jupyter_notebook_recipe import JupyterNotebookRecipe, PapermillHandler, \
getRecipeFromNotebook
from .python_recipe import PythonRecipe, PythonHandler

43
recipes/jupyter_notebook_recipe.py
View File

@ -11,21 +11,21 @@ import sys
from typing import Any, Tuple, Dict
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.meow import valid_event
from core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path, valid_existing_file_path
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
EVENT_RULE, get_base_file
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_notebook, write_notebook, \
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_notebook, write_notebook, \
write_yaml
from functionality.meow import create_job, replace_keywords
from meow_base.functionality.meow import create_job, replace_keywords
class JupyterNotebookRecipe(BaseRecipe):
@ -182,20 +182,37 @@ class PapermillHandler(BaseHandler):
# Send job directory, as actual definitons will be read from within it
self.to_runner.send(job_dir)
#TODO test me
def getRecipeFromNotebook(name:str, notebook_filename:str,
parameters:Dict[str,Any]={}, requirements:Dict[str,Any]={}
)->JupyterNotebookRecipe:
valid_existing_file_path(notebook_filename, extension=".ipynb")
check_type(name, str, hint="getRecipeFromNotebook.name")
notebook_code = read_notebook(notebook_filename)
return JupyterNotebookRecipe(
name,
notebook_code,
parameters=parameters,
requirements=requirements,
source=notebook_filename
)
# Papermill job execution code, to be run within the conductor
def papermill_job_func(job_dir):
# Requires own imports as will be run in its own execution environment
import os
import papermill
from datetime import datetime
from core.correctness.vars import JOB_EVENT, JOB_ID, \
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_job_file, \
get_result_file
from functionality.file_io import read_yaml, write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.parameterisation import parameterize_jupyter_notebook
from meow_base.functionality.file_io import read_yaml, write_notebook, write_yaml
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.parameterisation import parameterize_jupyter_notebook
# Identify job files

24
recipes/python_recipe.py
View File

@ -10,21 +10,21 @@ import sys
from typing import Any, Tuple, Dict, List
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.meow import valid_event
from core.correctness.validation import check_script, valid_string, \
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_script, valid_string, \
valid_dict, valid_dir_path
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
get_base_file
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_file_lines, write_file, \
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_file_lines, write_file, \
write_yaml, lines_to_string
from functionality.meow import create_job, replace_keywords
from meow_base.functionality.meow import create_job, replace_keywords
class PythonRecipe(BaseRecipe):
@ -180,14 +180,14 @@ def python_job_func(job_dir):
import os
from datetime import datetime
from io import StringIO
from core.correctness.vars import JOB_EVENT, JOB_ID, \
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_base_file, \
get_job_file, get_result_file
from functionality.file_io import read_yaml, write_yaml
from functionality.hashing import get_file_hash
from functionality.parameterisation import parameterize_python_script
from meow_base.functionality.file_io import read_yaml, write_yaml
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.parameterisation import parameterize_python_script
# Identify job files
meta_file = os.path.join(job_dir, META_FILE)

4
rules/__init__.py
View File

@ -1,3 +1,3 @@
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from rules.file_event_python_rule import FileEventPythonRule
from .file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from .file_event_python_rule import FileEventPythonRule

8
rules/file_event_jupyter_notebook_rule.py
View File

@ -6,10 +6,10 @@ and JupyterNotebookRecipe.
Author(s): David Marchant
"""
from core.base_rule import BaseRule
from core.correctness.validation import check_type
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
# TODO potentailly remove this and just invoke BaseRule directly, as does not
# add any functionality other than some validation.

8
rules/file_event_python_rule.py
View File

@ -6,10 +6,10 @@ and PythonRecipe.
Author(s): David Marchant
"""
from core.base_rule import BaseRule
from core.correctness.validation import check_type
from patterns.file_event_pattern import FileEventPattern
from recipes.python_recipe import PythonRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.python_recipe import PythonRecipe
# TODO potentailly remove this and just invoke BaseRule directly, as does not
# add any functionality other than some validation.

8
tests/shared.py
View File

@ -7,10 +7,10 @@ import os
from distutils.dir_util import copy_tree
from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from functionality.file_io import make_dir, rmtree
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from meow_base.functionality.file_io import make_dir, rmtree
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
# testing
TEST_DIR = "test_files"

16
tests/test_base.py
View File

@ -3,14 +3,14 @@ import unittest
from typing import Any, Union, Tuple, Dict
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from patterns import FileEventPattern
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from meow_base.patterns.file_event_pattern import FileEventPattern
from shared import setup, teardown, valid_pattern_one, valid_recipe_one

16
tests/test_conductors.py
View File

@ -5,21 +5,21 @@ import unittest
from datetime import datetime
from typing import Dict
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, JOB_EVENT, \
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, JOB_PATTERN, \
STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
from conductors import LocalPythonConductor
from functionality.file_io import read_file, read_yaml, write_file, \
from meow_base.conductors import LocalPythonConductor
from meow_base.functionality.file_io import read_file, read_yaml, write_file, \
write_notebook, write_yaml, lines_to_string, make_dir
from functionality.hashing import get_file_hash
from functionality.meow import create_watchdog_event, create_job, create_rule
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_watchdog_event, create_job, create_rule
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from recipes.python_recipe import PythonRecipe, python_job_func
from meow_base.recipes.python_recipe import PythonRecipe, python_job_func
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, \
BAREBONES_PYTHON_SCRIPT, BAREBONES_NOTEBOOK

26
tests/test_functionality.py
View File

@ -11,34 +11,34 @@ from sys import prefix, base_prefix
from time import sleep
from typing import Dict
from core.base_rule import BaseRule
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from functionality.debug import setup_debugging
from functionality.file_io import lines_to_string, make_dir, read_file, \
from meow_base.functionality.debug import setup_debugging
from meow_base.functionality.file_io import lines_to_string, make_dir, read_file, \
read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_event, create_job, create_rule, \
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_event, create_job, create_rule, \
create_rules, create_watchdog_event, replace_keywords, \
create_parameter_sweep, \
KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
KEYWORD_REL_PATH
from functionality.naming import _generate_id
from functionality.parameterisation import parameterize_jupyter_notebook, \
from meow_base.functionality.naming import _generate_id
from meow_base.functionality.parameterisation import parameterize_jupyter_notebook, \
parameterize_python_script
from functionality.process_io import wait
from functionality.requirements import create_python_requirements, \
from meow_base.functionality.process_io import wait
from meow_base.functionality.requirements import create_python_requirements, \
check_requirements, \
REQUIREMENT_PYTHON, REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, \
REQ_PYTHON_VERSION
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
@ -594,7 +594,7 @@ class MeowTests(unittest.TestCase):
self.assertEqual(len(rules), 0)
# Test that create_rules creates rules from patterns and recipes
# Test that create_rules creates rules from meow_base.patterns and recipes
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,

8
tests/test_patterns.py
View File

@ -5,13 +5,13 @@ import unittest
from multiprocessing import Pipe
from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
SWEEP_JUMP, SWEEP_STOP
from functionality.file_io import make_dir
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
from meow_base.functionality.file_io import make_dir
from meow_base.patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
_DEFAULT_MASK
from recipes import JupyterNotebookRecipe
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE

18
tests/test_recipes.py
View File

@ -6,23 +6,23 @@ import unittest
from multiprocessing import Pipe
from typing import Dict
from core.correctness.meow import valid_job
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
META_FILE, JOB_ERROR, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from functionality.file_io import lines_to_string, make_dir, read_yaml, \
from meow_base.functionality.file_io import lines_to_string, make_dir, read_yaml, \
write_file, write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_job, create_rules, create_rule, \
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_job, create_rules, create_rule, \
create_watchdog_event
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, papermill_job_func
from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from rules import FileEventJupyterNotebookRule, FileEventPythonRule
from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule
from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
COMPLETE_PYTHON_SCRIPT, TEST_JOB_QUEUE, TEST_MONITOR_BASE, \
TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK

6
tests/test_rules.py
View File

@ -1,9 +1,9 @@
import unittest
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from meow_base.rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from shared import setup, teardown, BAREBONES_NOTEBOOK
class CorrectnessTests(unittest.TestCase):

28
tests/test_runner.py
View File

@ -8,21 +8,21 @@ from random import shuffle
from shutil import copy
from time import sleep
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from conductors import LocalPythonConductor
from core.correctness.vars import get_result_file, \
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.conductors import LocalPythonConductor
from meow_base.core.correctness.vars import get_result_file, \
JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME
from core.runner import MeowRunner
from functionality.file_io import make_dir, read_file, read_notebook, \
from meow_base.core.runner import MeowRunner
from meow_base.functionality.file_io import make_dir, read_file, read_notebook, \
read_yaml, write_file, lines_to_string
from functionality.meow import create_parameter_sweep
from functionality.requirements import create_python_requirements
from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
from recipes.jupyter_notebook_recipe import PapermillHandler, \
from meow_base.functionality.meow import create_parameter_sweep
from meow_base.functionality.requirements import create_python_requirements
from meow_base.patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe
from recipes.python_recipe import PythonHandler, PythonRecipe
from meow_base.recipes.python_recipe import PythonHandler, PythonRecipe
from shared import setup, teardown, backup_before_teardown, \
TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, MAKER_RECIPE, \
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, FILTER_RECIPE, \
@ -916,10 +916,6 @@ class MeowTests(unittest.TestCase):
output = read_file(os.path.join(output_path))
self.assertEqual(output, "12505000.0")
def testSelfModifyingAnalysis(self)->None:
maker_pattern = FileEventPattern(
"maker_pattern",

10
tests/test_validation.py
View File

@ -5,16 +5,16 @@ import os
from datetime import datetime
from typing import Any, Union
from core.correctness.meow import valid_event, valid_job, valid_watchdog_event
from core.correctness.validation import check_type, check_implementation, \
from meow_base.core.correctness.meow import valid_event, valid_job, valid_watchdog_event
from meow_base.core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_dir_path, valid_non_existing_path, check_callable
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
WATCHDOG_HASH
from functionality.file_io import make_dir
from functionality.meow import create_rule
from meow_base.functionality.file_io import make_dir
from meow_base.functionality.meow import create_rule
from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \
valid_recipe_one