integrated threadsafe status updates

This commit is contained in:
PatchOfScotland
2023-04-18 13:50:20 +02:00
parent 3f28b11be9
commit ddca1f6aa4
11 changed files with 196 additions and 120 deletions

View File

@ -13,7 +13,8 @@ from os.path import exists, isfile, join
from typing import Any, Dict, List
from meow_base.core.vars import JOB_END_TIME, JOB_ERROR, JOB_STATUS, \
STATUS_FAILED, STATUS_DONE, JOB_CREATE_TIME, JOB_START_TIME
STATUS_FAILED, STATUS_DONE, JOB_CREATE_TIME, JOB_START_TIME, \
STATUS_SKIPPED, LOCK_EXT
from meow_base.functionality.validation import valid_path
@ -98,7 +99,7 @@ def write_yaml(source:Any, filename:str):
yaml.dump(source, param_file, default_flow_style=False)
def threadsafe_read_status(filepath:str):
lock_path = f"{filepath}.lock"
lock_path = filepath + LOCK_EXT
lock_handle = open(lock_path, 'a')
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
@ -113,7 +114,7 @@ def threadsafe_read_status(filepath:str):
return status
def threadsafe_write_status(source:dict[str,Any], filepath:str):
lock_path = f"{filepath}.lock"
lock_path = filepath + LOCK_EXT
lock_handle = open(lock_path, 'a')
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
@ -126,7 +127,7 @@ def threadsafe_write_status(source:dict[str,Any], filepath:str):
lock_handle.close()
def threadsafe_update_status(updates:dict[str,Any], filepath:str):
lock_path = f"{filepath}.lock"
lock_path = filepath + LOCK_EXT
lock_handle = open(lock_path, 'a')
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX)
@ -137,7 +138,7 @@ def threadsafe_update_status(updates:dict[str,Any], filepath:str):
if k in updates:
# Do not overwrite final job status
if k == JOB_STATUS \
and v in [STATUS_DONE, STATUS_FAILED]:
and v in [STATUS_DONE, STATUS_FAILED, STATUS_SKIPPED]:
continue
# Do not overwrite an existing time
elif k in [JOB_START_TIME, JOB_CREATE_TIME, JOB_END_TIME]:
@ -147,6 +148,10 @@ def threadsafe_update_status(updates:dict[str,Any], filepath:str):
updates[k] = f"{v} {updates[k]}"
status[k] = updates[k]
for k, v in updates.items():
if k not in status:
status[k] = v
write_yaml(status, filepath)
except Exception as e: