NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformated imports for pep8 compatability

Browse Source
This commit is contained in:
PatchOfScotland
2023-03-14 15:12:22 +01:00
parent af489d2bb9
commit ede29f3158
29 changed files with 225 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
benchmarking/mrme.py
View File

@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook
from shared import run_test, MRME
def multiple_rules_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
def multiple_rules_multiple_events(job_count:int, REPEATS:int, job_counter:int,
requested_jobs:int, runtime_start:float):
patterns = {}
for i in range(job_count):
pattern = FileEventPattern(

3
benchmarking/mrse.py
View File

@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook
from shared import run_test, MRSE
def multiple_rules_single_event(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
def multiple_rules_single_event(job_count:int, REPEATS:int, job_counter:int,
requested_jobs:int, runtime_start:float):
patterns = {}
for i in range(job_count):
pattern = FileEventPattern(

58
benchmarking/run_all.py
View File

@ -5,14 +5,18 @@ import sys
import time
import os
from shared import JOBS_COUNTS, REPEATS, TESTS, MRME, MRSE, SRME, SRSEP, SRSES, RESULTS_DIR, BASE, GRAPH_FILENAME
from typing import List
from shared import JOBS_COUNTS, REPEATS, TESTS, MRME, MRSE, SRME, SRSEP, \
SRSES, RESULTS_DIR, BASE
from mrme import multiple_rules_multiple_events
from mrse import multiple_rules_single_event
from srme import single_rule_multiple_events
from srsep import single_rule_single_event_parallel
from srsps import single_rule_single_event_sequential
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \
DEFAULT_JOB_QUEUE_DIR
from meow_base.functionality.file_io import rmtree
LINE_KEYS = {
@ -59,7 +63,7 @@ def run_tests():
print(f"All tests completed in: {str(time.time()-runtime_start)}")
def get_meow_graph(results_dir):
def get_meow_graph(results_dir:str):
lines = []
for run_type in os.listdir(results_dir):
@ -68,24 +72,25 @@ def get_meow_graph(results_dir):
# lines.append((f'scheduling {run_type}', [], 'solid'))
lines.append((run_type, [], 'solid'))
run_type_path = os.path.join(results_dir, run_type)
run_path = os.path.join(results_dir, run_type)
for job_count in os.listdir(run_type_path):
results_path = os.path.join(run_type_path, job_count, 'results.txt')
for job_count in os.listdir(run_path):
results_path = os.path.join(run_path, job_count, "results.txt")
with open(results_path, 'r') as f_in:
data = f_in.readlines()
scheduling_duration = 0
for line in data:
if "Average schedule time: " in line:
scheduling_duration = float(line.replace("Average schedule time: ", ''))
scheduling_duration = float(line.replace(
"Average schedule time: ", ''))
lines[-1][1].append((job_count, scheduling_duration))
lines[-1][1].sort(key=lambda y: float(y[0]))
return lines
def make_plot(lines, graph_path, title, logged):
def make_plot(lines:List, graph_path:str, title:str, logged:bool):
w = 10
h = 4
linecount = 0
@ -129,14 +134,19 @@ def make_plot(lines, graph_path, title, logged):
def make_both_plots(lines, path, title, log=True):
make_plot(lines, path, title, False)
if log:
logged_path = path[:path.index(".pdf")] + "_logged" + path[path.index(".pdf"):]
logged_path = path[:path.index(".pdf")] + "_logged" \
+ path[path.index(".pdf"):]
make_plot(lines, logged_path, title, True)
def make_graphs():
lines = get_meow_graph(RESULTS_DIR)
make_both_plots(lines, "result.pdf", "MiG scheduling overheads on the Threadripper")
make_both_plots(
lines,
"result.pdf",
"MiG scheduling overheads on the Threadripper"
)
average_lines = []
all_delta_lines = []
@ -146,20 +156,36 @@ def make_graphs():
averages = [(i, v/float(i)) for i, v in line_values]
average_lines.append((line_signature, averages, lines_style))
if line_signature not in ["total single_Pattern_single_file_sequential", "scheduling single_Pattern_single_file_sequential_jobs", "SPSFS"]:
if line_signature not in [
"total single_Pattern_single_file_sequential",
"scheduling single_Pattern_single_file_sequential_jobs",
"SPSFS"]:
deltas = []
for i in range(len(line_values)-1):
deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) )
deltas.append((line_values[i+1][0],
(averages[i+1][1]-averages[i][1]) \
/ (float(averages[i+1][0])-float(averages[i][0]))))
no_spsfs_delta_lines.append((line_signature, deltas, lines_style))
deltas = []
for i in range(len(line_values)-1):
deltas.append( (line_values[i+1][0], (averages[i+1][1]-averages[i][1]) / (float(averages[i+1][0])-float(averages[i][0])) ) )
deltas.append((line_values[i+1][0],
(averages[i+1][1]-averages[i][1]) \
/ (float(averages[i+1][0])-float(averages[i][0]))))
all_delta_lines.append((line_signature, deltas, lines_style))
make_both_plots(average_lines, "result_averaged.pdf", "Per-job MiG scheduling overheads on the Threadripper")
make_both_plots(
average_lines,
"result_averaged.pdf",
"Per-job MiG scheduling overheads on the Threadripper"
)
make_both_plots(all_delta_lines, "result_deltas.pdf", "Difference in per-job MiG scheduling overheads on the Threadripper", log=False)
make_both_plots(
all_delta_lines,
"result_deltas.pdf",
"Difference in per-job MiG scheduling overheads on the Threadripper",
log=False
)
if __name__ == '__main__':
try:
@ -172,4 +198,4 @@ if __name__ == '__main__':
try:
sys.exit(1)
except SystemExit:
os._exit(1)
os._exit(1)

85
benchmarking/shared.py
View File

@ -6,9 +6,12 @@ import pathlib
import time
import yaml
from typing import Any, Dict, Tuple
from typing import Any, Dict, Tuple, List
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \
DEFAULT_JOB_QUEUE_DIR
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.runner import MeowRunner
from meow_base.patterns.file_event_pattern import WatchdogMonitor
from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler
@ -17,9 +20,13 @@ from meow_base.functionality.file_io import rmtree
RESULTS_DIR = "results"
BASE = "benchmark_base"
GRAPH_FILENAME = "graph.pdf"
REPEATS = 10
JOBS_COUNTS = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200, 250, 300, 400, 500]
JOBS_COUNTS = [
10, 20, 30, 40, 50,
60, 70, 80, 90, 100,
125, 150, 175, 200,
250, 300, 400, 500
]
SRME = "single_rule_multiple_events"
MRSE = "multiple_rules_single_event"
@ -42,10 +49,11 @@ class DummyConductor(LocalPythonConductor):
return False, ">:("
def datetime_to_timestamp(date_time_obj):
return time.mktime(date_time_obj.timetuple()) + float(date_time_obj.microsecond)/1000000
def datetime_to_timestamp(date_time_obj:datetime):
return time.mktime(date_time_obj.timetuple()) \
+ float(date_time_obj.microsecond)/1000000
def generate(file_count, file_path, file_type='.txt'):
def generate(file_count:int, file_path:str, file_type:str=".txt"):
first_filename = ''
start = time.time()
for i in range(int(file_count)):
@ -56,7 +64,8 @@ def generate(file_count, file_path, file_type='.txt'):
f.write('0')
return first_filename, time.time() - start
def cleanup(jobs, file_out, base_time, gen_time, execution=False):
def cleanup(jobs:List[str], file_out:str, base_time:float, gen_time:float,
execution:bool=False):
if not jobs:
return
@ -116,27 +125,30 @@ def cleanup(jobs, file_out, base_time, gen_time, execution=False):
return total_time
def mean(l):
def mean(l:List):
return sum(l)/len(l)
def collate_results(base_results_dir):
def collate_results(base_dir:str):
scheduling_delays = []
for run in os.listdir(base_results_dir):
for run in os.listdir(base_dir):
if run != 'results.txt':
with open(os.path.join(base_results_dir, run, 'results.txt'), 'r') as f:
with open(os.path.join(base_dir, run, "results.txt"), 'r') as f:
d = f.readlines()
for l in d:
if "Total scheduling delay (seconds): " in l:
scheduling_delays.append(float(l.replace("Total scheduling delay (seconds): ", '')))
scheduling_delays.append(float(l.replace(
"Total scheduling delay (seconds): ", '')))
with open(os.path.join(base_results_dir, 'results.txt'), 'w') as f:
with open(os.path.join(base_dir, 'results.txt'), 'w') as f:
f.write(f"Average schedule time: {round(mean(scheduling_delays), 3)}\n")
f.write(f"Scheduling times: {scheduling_delays}")
def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_counter, requested_jobs, runtime_start, signature='', execution=False, print_logging=False):
def run_test(patterns:Dict[str,BasePattern], recipes:Dict[str,BaseRecipe],
files_count:int, expected_job_count:int, repeats:int, job_counter:int,
requested_jobs:int, runtime_start:float, signature:str="",
execution:bool=False, print_logging:bool=False):
if not os.path.exists(RESULTS_DIR):
os.mkdir(RESULTS_DIR)
@ -172,24 +184,12 @@ def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_co
print=runner_debug_stream,
logging=3
)
# meow.WorkflowRunner(
# VGRID,
# num_workers,
# patterns=patterns,
# recipes=recipes,
# daemon=True,
# start_workers=False,
# retro_active_jobs=False,
# print_logging=print_logging,
# file_logging=False,
# wait_time=1
# )
runner.start()
# Generate triggering files
first_filename, generation_duration = generate(files_count, file_base +"/file_")
first_filename, generation_duration = \
generate(files_count, file_base +"/file_")
idle_loops = 0
total_loops = 0
@ -216,10 +216,27 @@ def run_test(patterns, recipes, files_count, expected_job_count, repeats, job_co
else:
jobs = os.listdir(DEFAULT_JOB_QUEUE_DIR)
results_path = os.path.join(RESULTS_DIR, signature, str(expected_job_count), str(run), 'results.txt')
results_path = os.path.join(
RESULTS_DIR,
signature,
str(expected_job_count),
str(run),
"results.txt"
)
cleanup(jobs, results_path, first_filename, generation_duration, execution=execution)
cleanup(
jobs,
results_path,
first_filename,
generation_duration,
execution=execution
)
print(f"Completed scheduling run {str(run + 1)} of {str(len(jobs))}/{str(expected_job_count)} jobs for '{signature}' {job_counter + expected_job_count*(run+1)}/{requested_jobs} ({str(round(time.time()-runtime_start, 3))}s)")
print(f"Completed scheduling run {str(run + 1)} of {str(len(jobs))}"
f"/{str(expected_job_count)} jobs for '{signature}' "
f"{job_counter + expected_job_count*(run+1)}/{requested_jobs} "
f"({str(round(time.time()-runtime_start, 3))}s)")
collate_results(os.path.join(RESULTS_DIR, signature, str(expected_job_count)))
collate_results(
os.path.join(RESULTS_DIR, signature, str(expected_job_count))
)

3
benchmarking/srme.py
View File

@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook
from shared import run_test, SRME
def single_rule_multiple_events(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
def single_rule_multiple_events(job_count:int, REPEATS:int, job_counter:int,
requested_jobs:int, runtime_start:float):
patterns = {}
pattern = FileEventPattern(
f"pattern_one",

3
benchmarking/srsep.py
View File

@ -5,7 +5,8 @@ from meow_base.recipes import get_recipe_from_notebook
from meow_base.functionality.meow import create_parameter_sweep
from shared import run_test, SRSEP
def single_rule_single_event_parallel(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
def single_rule_single_event_parallel(job_count:int, REPEATS:int,
job_counter:int, requested_jobs:int, runtime_start:float):
patterns = {}
pattern = FileEventPattern(
f"pattern_one",

3
benchmarking/srsps.py
View File

@ -4,7 +4,8 @@ from meow_base.recipes import get_recipe_from_notebook
from shared import run_test, SRSES
def single_rule_single_event_sequential(job_count:int, REPEATS, job_counter, requested_jobs, runtime_start):
def single_rule_single_event_sequential(job_count:int, REPEATS:int,
job_counter:int, requested_jobs:int, runtime_start:float):
patterns = {}
pattern = FileEventPattern(
f"pattern_one",

12
conductors/local_python_conductor.py
View File

@ -13,12 +13,14 @@ from typing import Any, Tuple, Dict
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
STATUS_RUNNING, JOB_START_TIME, META_FILE, BACKUP_JOB_ERROR_FILE, \
STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, \
JOB_STATUS, STATUS_RUNNING, JOB_START_TIME, META_FILE, \
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
JOB_ERROR, JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, \
DEFAULT_JOB_OUTPUT_DIR
from meow_base.core.correctness.validation import valid_dir_path
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, write_yaml
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
write_yaml
class LocalPythonConductor(BaseConductor):
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR,

2
core/base_handler.py
View File

@ -8,7 +8,7 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation

2
core/base_monitor.py
View File

@ -12,7 +12,7 @@ from typing import Union, Dict
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_CHANNELS
from meow_base.core.correctness.vars import VALID_CHANNELS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation
from meow_base.functionality.meow import create_rules

4
core/base_pattern.py
View File

@ -10,8 +10,8 @@ import itertools
from typing import Any, Union, Tuple, Dict, List
from meow_base.core.correctness.vars import get_drt_imp_msg, \
VALID_PATTERN_NAME_CHARS, SWEEP_JUMP, SWEEP_START, SWEEP_STOP
from meow_base.core.correctness.vars import VALID_PATTERN_NAME_CHARS, \
SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation, valid_dict

6
core/base_recipe.py
View File

@ -8,8 +8,10 @@ Author(s): David Marchant
from typing import Any, Dict
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RECIPE_NAME_CHARS
from meow_base.core.correctness.validation import valid_string, check_implementation
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
valid_string
class BaseRecipe:

3
core/base_rule.py
View File

@ -13,7 +13,8 @@ if "BasePattern" not in modules:
from meow_base.core.base_pattern import BasePattern
if "BaseRecipe" not in modules:
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.vars import get_drt_imp_msg, VALID_RULE_NAME_CHARS
from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import valid_string, check_type, \
check_implementation

6
core/correctness/meow.py
View File

@ -4,9 +4,9 @@ from typing import Any, Dict, Type
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type
from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, JOB_EVENT, \
JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, \
JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \
JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
# Required keys in event dict
EVENT_KEYS = {

9
core/runner.py
View File

@ -17,10 +17,11 @@ from typing import Any, Union, Dict, List
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, EVENT_TYPE, \
VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR, \
EVENT_PATH
from meow_base.core.correctness.validation import check_type, valid_list, valid_dir_path
from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, \
EVENT_TYPE, VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, \
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH
from meow_base.core.correctness.validation import check_type, valid_list, \
valid_dir_path
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_yaml
from meow_base.functionality.process_io import wait

3
functionality/hashing.py
View File

@ -7,7 +7,8 @@ Author(s): David Marchant
from hashlib import sha256
from meow_base.core.correctness.vars import HASH_BUFFER_SIZE, SHA256
from meow_base.core.correctness.validation import check_type, valid_existing_file_path
from meow_base.core.correctness.validation import check_type, \
valid_existing_file_path
def _get_file_sha256(file_path):
sha256_hash = sha256()

13
functionality/meow.py
View File

@ -11,12 +11,13 @@ from typing import Any, Dict, Union, List
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type, valid_dict, valid_list
from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, JOB_TYPE, \
STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, SWEEP_START, \
SWEEP_STOP
from meow_base.core.correctness.validation import check_type, valid_dict, \
valid_list
from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \
EVENT_TYPE, EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \
JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \
SWEEP_START, SWEEP_STOP
from meow_base.functionality.naming import generate_job_id, generate_rule_id
# mig trigger keyword replacements

25
patterns/file_event_pattern.py
View File

@ -23,7 +23,7 @@ from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.validation import check_type, valid_string, \
valid_dict, valid_list, valid_path, valid_dir_path
valid_dict, valid_list, valid_dir_path
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, \
@ -580,29 +580,6 @@ class WatchdogEventHandler(PatternMatchingEventHandler):
self.monitor.match(event)
# recent_timestamp = self._recent_jobs[event.src_path]
# difference = event.time_stamp - recent_timestamp
#
# # Discard the event if we already have a recent event at this
# # same path. Update the most recent time, so we can hopefully
# # wait till events have stopped happening
# if difference <= self._settletime:
# self._recent_jobs[event.src_path] = \
# max(recent_timestamp, event.time_stamp)
# self._recent_jobs_lock.release()
# return
# else:
# self._recent_jobs[event.src_path] = event.time_stamp
# else:
# self._recent_jobs[event.src_path] = event.time_stamp
# except Exception as ex:
# self._recent_jobs_lock.release()
# raise Exception(ex)
# self._recent_jobs_lock.release()
#
# # If we did not have a recent event, then send it on to the monitor
# self.monitor.match(event)
def handle_event(self, event):
"""Handler function, called by all specific event functions. Will
attach a timestamp to the event immediately, and attempt to start a

16
recipes/jupyter_notebook_recipe.py
View File

@ -16,15 +16,15 @@ from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path, valid_existing_file_path
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
EVENT_RULE, get_base_file
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, \
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, \
JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, PARAMS_FILE, \
JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, EVENT_RULE, \
get_base_file
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_notebook, write_notebook, \
write_yaml
from meow_base.functionality.file_io import make_dir, read_notebook, \
write_notebook, write_yaml
from meow_base.functionality.meow import create_job, replace_keywords

12
recipes/python_recipe.py
View File

@ -15,15 +15,15 @@ from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_script, valid_string, \
valid_dict, valid_dir_path
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, \
DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, \
WATCHDOG_HASH, JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
get_base_file
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_file_lines, write_file, \
write_yaml, lines_to_string
from meow_base.functionality.file_io import make_dir, read_file_lines, \
write_file, write_yaml, lines_to_string
from meow_base.functionality.meow import create_job, replace_keywords

3
tests/shared.py
View File

@ -7,7 +7,8 @@ import os
from distutils.dir_util import copy_tree
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \
DEFAULT_JOB_QUEUE_DIR
from meow_base.functionality.file_io import make_dir, rmtree
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe

2
tests/test_base.py
View File

@ -11,7 +11,7 @@ from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from meow_base.patterns.file_event_pattern import FileEventPattern
from shared import setup, teardown, valid_pattern_one, valid_recipe_one
from shared import valid_pattern_one, valid_recipe_one, setup, teardown
class BaseRecipeTests(unittest.TestCase):

21
tests/test_conductors.py
View File

@ -5,24 +5,25 @@ import unittest
from datetime import datetime
from typing import Dict
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, JOB_EVENT, \
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, JOB_PATTERN, \
STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, \
JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, \
JOB_EVENT, META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \
JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \
JOB_CREATE_TIME, JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
from meow_base.conductors import LocalPythonConductor
from meow_base.functionality.file_io import read_file, read_yaml, write_file, \
write_notebook, write_yaml, lines_to_string, make_dir
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_watchdog_event, create_job, create_rule
from meow_base.functionality.meow import create_watchdog_event, create_job, \
create_rule
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from meow_base.recipes.python_recipe import PythonRecipe, python_job_func
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, \
BAREBONES_PYTHON_SCRIPT, BAREBONES_NOTEBOOK
from shared import TEST_MONITOR_BASE, APPENDING_NOTEBOOK, TEST_JOB_OUTPUT, \
TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, BAREBONES_PYTHON_SCRIPT, \
BAREBONES_NOTEBOOK, setup, teardown
def failing_func():
raise Exception("bad function")
@ -129,7 +130,7 @@ class MeowTests(unittest.TestCase):
self.assertTrue(os.path.exists(result_path))
# Test LocalPythonConductor executes valid papermill jobs
# Test LocalPythonConductor executes valid papermill jobs
def testLocalPythonConductorValidPapermillJob(self)->None:
lpc = LocalPythonConductor(
job_queue_dir=TEST_JOB_QUEUE,

32
tests/test_functionality.py
View File

@ -19,29 +19,27 @@ from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from meow_base.functionality.debug import setup_debugging
from meow_base.functionality.file_io import lines_to_string, make_dir, read_file, \
read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
from meow_base.functionality.file_io import lines_to_string, make_dir, \
read_file, read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
write_notebook, write_yaml
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_event, create_job, create_rule, \
create_rules, create_watchdog_event, replace_keywords, \
create_parameter_sweep, \
KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
KEYWORD_REL_PATH
from meow_base.functionality.meow import KEYWORD_BASE, KEYWORD_DIR, \
KEYWORD_EXTENSION, KEYWORD_FILENAME, KEYWORD_JOB, KEYWORD_PATH, \
KEYWORD_PREFIX, KEYWORD_REL_DIR, KEYWORD_REL_PATH, \
create_event, create_job, create_rule, create_rules, \
create_watchdog_event, replace_keywords, create_parameter_sweep
from meow_base.functionality.naming import _generate_id
from meow_base.functionality.parameterisation import parameterize_jupyter_notebook, \
parameterize_python_script
from meow_base.functionality.parameterisation import \
parameterize_jupyter_notebook, parameterize_python_script
from meow_base.functionality.process_io import wait
from meow_base.functionality.requirements import create_python_requirements, \
check_requirements, \
REQUIREMENT_PYTHON, REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, \
REQ_PYTHON_VERSION
from meow_base.functionality.requirements import REQUIREMENT_PYTHON, \
REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, REQ_PYTHON_VERSION, \
create_python_requirements, check_requirements
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
from shared import TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, \
COMPLETE_PYTHON_SCRIPT, valid_recipe_two, valid_recipe_one, \
valid_pattern_one, valid_pattern_two, setup, teardown
class DebugTests(unittest.TestCase):
def setUp(self)->None:

6
tests/test_patterns.py
View File

@ -9,10 +9,10 @@ from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
SWEEP_JUMP, SWEEP_STOP
from meow_base.functionality.file_io import make_dir
from meow_base.patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
_DEFAULT_MASK
from meow_base.patterns.file_event_pattern import FileEventPattern, \
WatchdogMonitor, _DEFAULT_MASK
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE
from shared import BAREBONES_NOTEBOOK, TEST_MONITOR_BASE, setup, teardown
def patterns_equal(tester, pattern_one, pattern_two):

30
tests/test_recipes.py
View File

@ -7,26 +7,26 @@ from multiprocessing import Pipe
from typing import Dict
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
META_FILE, JOB_ERROR, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from meow_base.functionality.file_io import lines_to_string, make_dir, read_yaml, \
write_file, write_notebook, write_yaml
from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, \
EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \
JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, \
JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \
SWEEP_START, JOB_TYPE_PAPERMILL, get_base_file, get_job_file, \
get_result_file
from meow_base.functionality.file_io import lines_to_string, make_dir, \
read_yaml, write_file, write_notebook, write_yaml
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_job, create_rules, create_rule, \
create_watchdog_event
from meow_base.functionality.meow import create_job, create_rules, \
create_rule, create_watchdog_event
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, papermill_job_func, get_recipe_from_notebook
from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, \
python_job_func
from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule
from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
COMPLETE_PYTHON_SCRIPT, TEST_JOB_QUEUE, TEST_MONITOR_BASE, \
TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK
from shared import BAREBONES_PYTHON_SCRIPT, COMPLETE_PYTHON_SCRIPT, \
TEST_JOB_QUEUE, TEST_MONITOR_BASE, TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, \
APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK, setup, teardown
class JupyterNotebookTests(unittest.TestCase):
def setUp(self)->None:

5
tests/test_rules.py
View File

@ -3,8 +3,9 @@ import unittest
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from meow_base.rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from shared import setup, teardown, BAREBONES_NOTEBOOK
from meow_base.rules.file_event_jupyter_notebook_rule import \
FileEventJupyterNotebookRule
from shared import BAREBONES_NOTEBOOK, setup, teardown
class CorrectnessTests(unittest.TestCase):
def setUp(self)->None:

22
tests/test_runner.py
View File

@ -12,23 +12,23 @@ from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.conductors import LocalPythonConductor
from meow_base.core.correctness.vars import get_result_file, \
JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME
from meow_base.core.correctness.vars import JOB_TYPE_PAPERMILL, JOB_ERROR, \
META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME, get_result_file
from meow_base.core.runner import MeowRunner
from meow_base.functionality.file_io import make_dir, read_file, read_notebook, \
read_yaml, write_file, lines_to_string
from meow_base.functionality.file_io import make_dir, read_file, \
read_notebook, read_yaml, write_file, lines_to_string
from meow_base.functionality.meow import create_parameter_sweep
from meow_base.functionality.requirements import create_python_requirements
from meow_base.patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
from meow_base.patterns.file_event_pattern import WatchdogMonitor, \
FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe
from meow_base.recipes.python_recipe import PythonHandler, PythonRecipe
from shared import setup, teardown, backup_before_teardown, \
TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, MAKER_RECIPE, \
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, FILTER_RECIPE, \
POROSITY_CHECK_NOTEBOOK, SEGMENT_FOAM_NOTEBOOK, GENERATOR_NOTEBOOK, \
FOAM_PORE_ANALYSIS_NOTEBOOK, IDMC_UTILS_MODULE, TEST_DATA, GENERATE_SCRIPT
from shared import TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, \
MAKER_RECIPE, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, \
FILTER_RECIPE, POROSITY_CHECK_NOTEBOOK, SEGMENT_FOAM_NOTEBOOK, \
GENERATOR_NOTEBOOK, FOAM_PORE_ANALYSIS_NOTEBOOK, IDMC_UTILS_MODULE, \
TEST_DATA, GENERATE_SCRIPT, setup, teardown, backup_before_teardown
pattern_check = FileEventPattern(
"pattern_check",

22
tests/test_validation.py
View File

@ -5,18 +5,20 @@ import os
from datetime import datetime
from typing import Any, Union
from meow_base.core.correctness.meow import valid_event, valid_job, valid_watchdog_event
from meow_base.core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_dir_path, valid_non_existing_path, check_callable
from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
WATCHDOG_HASH
from meow_base.core.correctness.meow import valid_event, valid_job, \
valid_watchdog_event
from meow_base.core.correctness.validation import check_type, \
check_implementation, valid_string, valid_dict, valid_list, \
valid_existing_file_path, valid_dir_path, valid_non_existing_path, \
check_callable
from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, \
EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
WATCHDOG_BASE, WATCHDOG_HASH
from meow_base.functionality.file_io import make_dir
from meow_base.functionality.meow import create_rule
from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \
valid_recipe_one
from shared import TEST_MONITOR_BASE, valid_pattern_one, valid_recipe_one, \
setup, teardown
class ValidationTests(unittest.TestCase):
def setUp(self)->None: