NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

integrated threadsafe status updates

Browse Source
This commit is contained in:
PatchOfScotland
2023-04-18 13:50:20 +02:00
parent 3f28b11be9
commit ddca1f6aa4
11 changed files with 196 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
recipes/jupyter_notebook_recipe.py
View File

@ -24,7 +24,8 @@ from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
get_base_file
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_notebook, \
write_notebook, write_yaml
write_notebook, write_yaml, threadsafe_write_status, \
threadsafe_update_status
from meow_base.functionality.meow import create_job, replace_keywords
@ -163,7 +164,7 @@ class PapermillHandler(BaseHandler):
# write a status file to the job directory
meta_file = os.path.join(job_dir, META_FILE)
write_yaml(meow_job, meta_file)
threadsafe_write_status(meow_job, meta_file)
# write an executable notebook to the job directory
base_file = os.path.join(job_dir, get_base_file(JOB_TYPE_PAPERMILL))
@ -176,7 +177,12 @@ class PapermillHandler(BaseHandler):
meow_job[JOB_STATUS] = STATUS_QUEUED
# update the status file with queued status
write_yaml(meow_job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_QUEUED
},
meta_file
)
# Send job directory, as actual definitons will be read from within it
self.to_runner.send(job_dir)
@ -208,7 +214,8 @@ def papermill_job_func(job_dir):
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_job_file, \
get_result_file
from meow_base.functionality.file_io import read_yaml, write_notebook, write_yaml
from meow_base.functionality.file_io import read_yaml, write_notebook, \
threadsafe_read_status, threadsafe_update_status
from meow_base.functionality.hashing import get_hash
from meow_base.functionality.parameterisation import parameterize_jupyter_notebook
@ -221,7 +228,7 @@ def papermill_job_func(job_dir):
param_file = os.path.join(job_dir, PARAMS_FILE)
# Get job defintions
job = read_yaml(meta_file)
job = threadsafe_read_status(meta_file)
yaml_dict = read_yaml(param_file)
# Check the hash of the triggering file, if present. This addresses
@ -234,15 +241,20 @@ def papermill_job_func(job_dir):
# another job will have been scheduled anyway.
if not triggerfile_hash \
or triggerfile_hash != job[JOB_EVENT][WATCHDOG_HASH]:
job[JOB_STATUS] = STATUS_SKIPPED
job[JOB_END_TIME] = datetime.now()
msg = "Job was skipped as triggering file " + \
f"'{job[JOB_EVENT][EVENT_PATH]}' has been modified since " + \
"scheduling. Was expected to have hash " + \
f"'{job[JOB_EVENT][WATCHDOG_HASH]}' but has '" + \
f"{triggerfile_hash}'."
job[JOB_ERROR] = msg
write_yaml(job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_SKIPPED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: msg
},
meta_file
)
return
# Create a parameterised version of the executable notebook
@ -253,20 +265,29 @@ def papermill_job_func(job_dir):
)
write_notebook(job_notebook, job_file)
except Exception as e:
job[JOB_STATUS] = STATUS_FAILED
job[JOB_END_TIME] = datetime.now()
msg = f"Job file {job[JOB_ID]} was not created successfully. {e}"
job[JOB_ERROR] = msg
write_yaml(job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_FAILED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: msg
},
meta_file
)
return
# Execute the parameterised notebook
try:
papermill.execute_notebook(job_file, result_file, {})
except Exception as e:
job[JOB_STATUS] = STATUS_FAILED
job[JOB_END_TIME] = datetime.now()
msg = f"Result file {result_file} was not created successfully. {e}"
job[JOB_ERROR] = msg
write_yaml(job, meta_file)
threadsafe_update_status(
{
JOB_STATUS: STATUS_FAILED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: msg
},
meta_file
)
return