NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

integrated threadsafe status updates

Browse Source
This commit is contained in:
PatchOfScotland
2023-04-18 13:50:20 +02:00
parent 3f28b11be9
commit ddca1f6aa4
11 changed files with 196 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
recipes/python_recipe.py
View File

@ -23,7 +23,8 @@ from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
get_base_file
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_file_lines, \
write_file, write_yaml, lines_to_string
write_file, write_yaml, lines_to_string, threadsafe_write_status, \
threadsafe_update_status
from meow_base.functionality.meow import create_job, replace_keywords
@ -153,7 +154,7 @@ class PythonHandler(BaseHandler):
# write a status file to the job directory
meta_file = os.path.join(job_dir, META_FILE)
write_yaml(meow_job, meta_file)
threadsafe_write_status(meow_job, meta_file)
# write an executable script to the job directory
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PYTHON))
@ -163,10 +164,13 @@ class PythonHandler(BaseHandler):
param_file = os.path.join(job_dir, PARAMS_FILE)
write_yaml(yaml_dict, param_file)
meow_job[JOB_STATUS] = STATUS_QUEUED
# update the status file with queued status
write_yaml(meow_job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_QUEUED
},
meta_file
)
# Send job directory, as actual definitons will be read from within it
self.to_runner.send(job_dir)
@ -184,7 +188,8 @@ def python_job_func(job_dir):
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_base_file, \
get_job_file, get_result_file
from meow_base.functionality.file_io import read_yaml, write_yaml
from meow_base.functionality.file_io import read_yaml, \
threadsafe_read_status, threadsafe_update_status
from meow_base.functionality.hashing import get_hash
from meow_base.functionality.parameterisation import parameterize_python_script
@ -196,7 +201,7 @@ def python_job_func(job_dir):
param_file = os.path.join(job_dir, PARAMS_FILE)
# Get job defintions
job = read_yaml(meta_file)
job = threadsafe_read_status(meta_file)
yaml_dict = read_yaml(param_file)
# Check the hash of the triggering file, if present. This addresses
@ -209,15 +214,19 @@ def python_job_func(job_dir):
# another job will have been scheduled anyway.
if not triggerfile_hash \
or triggerfile_hash != job[JOB_EVENT][WATCHDOG_HASH]:
job[JOB_STATUS] = STATUS_SKIPPED
job[JOB_END_TIME] = datetime.now()
msg = "Job was skipped as triggering file " + \
f"'{job[JOB_EVENT][EVENT_PATH]}' has been modified since " + \
"scheduling. Was expected to have hash " + \
f"'{job[JOB_EVENT][WATCHDOG_HASH]}' but has '" + \
f"{triggerfile_hash}'."
job[JOB_ERROR] = msg
write_yaml(job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_SKIPPED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: msg
},
meta_file
)
return
# Create a parameterised version of the executable script
@ -228,11 +237,15 @@ def python_job_func(job_dir):
)
write_file(lines_to_string(job_script), job_file)
except Exception as e:
job[JOB_STATUS] = STATUS_FAILED
job[JOB_END_TIME] = datetime.now()
msg = f"Job file {job[JOB_ID]} was not created successfully. {e}"
job[JOB_ERROR] = msg
write_yaml(job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_FAILED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: msg
},
meta_file
)
return
# Execute the parameterised script
@ -256,11 +269,16 @@ def python_job_func(job_dir):
sys.stdout = std_stdout
sys.stderr = std_stderr
job[JOB_STATUS] = STATUS_FAILED
job[JOB_END_TIME] = datetime.now()
msg = f"Result file {result_file} was not created successfully. {e}"
job[JOB_ERROR] = msg
write_yaml(job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_FAILED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: msg
},
meta_file
)
return
sys.stdout = std_stdout