NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

integrated threadsafe status updates

Browse Source
This commit is contained in:
PatchOfScotland
2023-04-18 13:50:20 +02:00
parent 3f28b11be9
commit ddca1f6aa4
11 changed files with 196 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
tests/shared.py
View File

@ -6,9 +6,10 @@ Author(s): David Marchant
import os
from distutils.dir_util import copy_tree
from typing import List
from meow_base.core.vars import DEFAULT_JOB_OUTPUT_DIR, \
DEFAULT_JOB_QUEUE_DIR
DEFAULT_JOB_QUEUE_DIR, LOCK_EXT
from meow_base.functionality.file_io import make_dir, rmtree
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
@ -37,7 +38,9 @@ def teardown():
rmtree(DEFAULT_JOB_QUEUE_DIR)
rmtree("first")
for f in [
"temp_phantom_info.h5", "temp_phantom.h5", "doesNotExist.lock"
"temp_phantom_info.h5",
"temp_phantom.h5",
f"doesNotExist{LOCK_EXT}"
]:
if os.path.exists(f):
os.remove(f)
@ -46,6 +49,14 @@ def backup_before_teardown(backup_source:str, backup_dest:str):
make_dir(backup_dest, ensure_clean=True)
copy_tree(backup_source, backup_dest)
# Necessary, as the creation of locks is not deterministic
def list_non_locks(dir:str)->List[str]:
return [f for f in os.listdir(dir) if not f.endswith(LOCK_EXT)]
# Necessary, as the creation of locks is not deterministic
def count_non_locks(dir:str)->int:
return len(list_non_locks(dir))
# Bash scripts
BAREBONES_BASH_SCRIPT = [

32
tests/test_functionality.py
View File

@ -13,7 +13,7 @@ from typing import Dict
from meow_base.core.rule import Rule
from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, LOCK_EXT, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, JOB_ERROR, STATUS_DONE, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
@ -348,7 +348,7 @@ data"""
self.assertFalse(os.path.exists(filepath))
threadsafe_write_status(first_yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
self.assertTrue(os.path.exists(f"{filepath}.lock"))
self.assertTrue(os.path.exists(filepath + LOCK_EXT))
with open(filepath, 'r') as f:
data = f.readlines()
@ -381,7 +381,7 @@ data"""
threadsafe_write_status(second_yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
self.assertTrue(os.path.exists(f"{filepath}.lock"))
self.assertTrue(os.path.exists(filepath + LOCK_EXT))
with open(filepath, 'r') as f:
data = f.readlines()
@ -444,7 +444,7 @@ data"""
self.assertFalse(os.path.exists(filepath))
threadsafe_write_status(first_yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
self.assertTrue(os.path.exists(f"{filepath}.lock"))
self.assertTrue(os.path.exists(filepath + LOCK_EXT))
with open(filepath, 'r') as f:
data = f.readlines()
@ -475,7 +475,7 @@ data"""
threadsafe_update_status(second_yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
self.assertTrue(os.path.exists(f"{filepath}.lock"))
self.assertTrue(os.path.exists(filepath + LOCK_EXT))
with open(filepath, 'r') as f:
data = f.readlines()
@ -498,7 +498,8 @@ data"""
JOB_CREATE_TIME: "now",
JOB_STATUS: "Wham",
JOB_ERROR: "first error.",
JOB_ID: "id"
JOB_ID: "id",
JOB_TYPE: "type"
}
filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
@ -506,7 +507,7 @@ data"""
self.assertFalse(os.path.exists(filepath))
threadsafe_write_status(first_yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
self.assertTrue(os.path.exists(f"{filepath}.lock"))
self.assertTrue(os.path.exists(filepath + LOCK_EXT))
status = threadsafe_read_status(filepath)
@ -523,7 +524,7 @@ data"""
threadsafe_update_status(second_yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
self.assertTrue(os.path.exists(f"{filepath}.lock"))
self.assertTrue(os.path.exists(filepath + LOCK_EXT))
status = threadsafe_read_status(filepath)
@ -531,7 +532,8 @@ data"""
JOB_CREATE_TIME: "now",
JOB_STATUS: STATUS_DONE,
JOB_ERROR: "first error. changed.",
JOB_ID: "changed"
JOB_ID: "changed",
JOB_TYPE: "type"
}
self.assertEqual(expected_second_yaml_dict, status)
@ -540,14 +542,15 @@ data"""
JOB_CREATE_TIME: "editted",
JOB_STATUS: "editted",
JOB_ERROR: "editted.",
JOB_ID: "editted"
JOB_ID: "editted",
"something_new": "new"
}
filepath = os.path.join(TEST_MONITOR_BASE, "file.yaml")
threadsafe_update_status(third_yaml_dict, filepath)
self.assertTrue(os.path.exists(filepath))
self.assertTrue(os.path.exists(f"{filepath}.lock"))
self.assertTrue(os.path.exists(filepath + LOCK_EXT))
status = threadsafe_read_status(filepath)
@ -555,8 +558,13 @@ data"""
JOB_CREATE_TIME: "now",
JOB_STATUS: STATUS_DONE,
JOB_ERROR: "first error. changed. editted.",
JOB_ID: "editted"
JOB_ID: "editted",
JOB_TYPE: "type",
"something_new": "new"
}
print(expected_third_yaml_dict)
print(status)
self.assertEqual(expected_third_yaml_dict, status)

14
tests/test_runner.py
View File

@ -29,7 +29,8 @@ from shared import TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, \
MAKER_RECIPE, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, \
FILTER_RECIPE, POROSITY_CHECK_NOTEBOOK, SEGMENT_FOAM_NOTEBOOK, \
GENERATOR_NOTEBOOK, FOAM_PORE_ANALYSIS_NOTEBOOK, IDMC_UTILS_PYTHON_SCRIPT, \
TEST_DATA, GENERATE_PYTHON_SCRIPT, setup, teardown, backup_before_teardown
TEST_DATA, GENERATE_PYTHON_SCRIPT, \
setup, teardown, backup_before_teardown, count_non_locks
pattern_check = FileEventPattern(
"pattern_check",
@ -329,7 +330,8 @@ class MeowTests(unittest.TestCase):
runner.stop()
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(job_dir)), 5)
print(os.listdir(job_dir))
self.assertEqual(count_non_locks(job_dir), 5)
result = read_notebook(
os.path.join(job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
@ -426,7 +428,7 @@ class MeowTests(unittest.TestCase):
runner.stop()
mid_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(mid_job_dir)), 5)
self.assertEqual(count_non_locks(mid_job_dir), 5)
result = read_notebook(
os.path.join(mid_job_dir, get_result_file(JOB_TYPE_PAPERMILL)))
@ -441,7 +443,7 @@ class MeowTests(unittest.TestCase):
self.assertEqual(data, "Initial Data\nA line from Pattern 1")
final_job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
self.assertEqual(len(os.listdir(final_job_dir)), 5)
self.assertEqual(count_non_locks(final_job_dir), 5)
result = read_notebook(os.path.join(final_job_dir,
get_result_file(JOB_TYPE_PAPERMILL)))
@ -645,7 +647,7 @@ class MeowTests(unittest.TestCase):
final_job_id = job_ids[0]
mid_job_dir = os.path.join(TEST_JOB_OUTPUT, mid_job_id)
self.assertEqual(len(os.listdir(mid_job_dir)), 5)
self.assertEqual(count_non_locks(mid_job_dir), 5)
mid_metafile = os.path.join(mid_job_dir, META_FILE)
mid_status = read_yaml(mid_metafile)
@ -664,7 +666,7 @@ class MeowTests(unittest.TestCase):
self.assertEqual(mid_output, "7806.25")
final_job_dir = os.path.join(TEST_JOB_OUTPUT, final_job_id)
self.assertEqual(len(os.listdir(final_job_dir)), 5)
self.assertEqual(count_non_locks(final_job_dir), 5)
final_metafile = os.path.join(final_job_dir, META_FILE)
final_status = read_yaml(final_metafile)