NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformted jobs being passed to conductors so they only get a job directory and have to read the definitions from the appropriate files

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-09 15:22:26 +01:00
parent 1eb022f79e
commit a2df62c693
19 changed files with 528 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
conductors/local_python_conductor.py
View File

@ -12,20 +12,26 @@ from datetime import datetime
from typing import Any, Tuple, Dict
from core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, JOB_STATUS, \
STATUS_RUNNING, JOB_START_TIME, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, PYTHON_OUTPUT_DIR, \
JOB_TYPE, JOB_TYPE_PAPERMILL
from core.correctness.validation import valid_job
from core.functionality import read_yaml, write_yaml
STATUS_RUNNING, JOB_START_TIME, JOB_ID, META_FILE, \
STATUS_DONE, JOB_END_TIME, STATUS_FAILED, JOB_ERROR, \
JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
from core.correctness.validation import valid_job, valid_dir_path
from core.functionality import read_yaml, write_yaml, make_dir
from core.meow import BaseConductor
class LocalPythonConductor(BaseConductor):
def __init__(self)->None:
def __init__(self, job_queue_dir:str=DEFAULT_JOB_QUEUE_DIR,
job_output_dir:str=DEFAULT_JOB_OUTPUT_DIR)->None:
"""LocalPythonConductor Constructor. This should be used to execute
Python jobs, and will then pass any internal job runner files to the
output directory."""
output directory. Note that if this handler is given to a MeowRunner
object, the job_queue_dir and job_output_dir will be overwridden."""
super().__init__()
self._is_valid_job_queue_dir(job_queue_dir)
self.job_queue_dir = job_queue_dir
self._is_valid_job_output_dir(job_output_dir)
self.job_output_dir = job_output_dir
def valid_execute_criteria(self, job:Dict[str,Any])->Tuple[bool,str]:
"""Function to determine given an job defintion, if this conductor can
@ -38,16 +44,17 @@ class LocalPythonConductor(BaseConductor):
pass
return False, str(e)
def execute(self, job:Dict[str,Any])->None:
def execute(self, job_dir:str)->None:
"""Function to actually execute a Python job. This will read job
defintions from its meta file, update the meta file and attempt to
execute. Some unspecific feedback will be given on execution failure,
but depending on what it is it may be up to the job itself to provide
more detailed feedback."""
valid_job(job)
valid_dir_path(job_dir, must_exist=True)
job_dir = os.path.join(job[PYTHON_EXECUTION_BASE], job[JOB_ID])
meta_file = os.path.join(job_dir, META_FILE)
job = read_yaml(meta_file)
valid_job(job)
# update the status file with running status
job[JOB_STATUS] = STATUS_RUNNING
@ -57,7 +64,7 @@ class LocalPythonConductor(BaseConductor):
# execute the job
try:
job_function = job[PYTHON_FUNC]
job_function(job)
job_function(job_dir)
# get up to date job data
job = read_yaml(meta_file)
@ -83,5 +90,20 @@ class LocalPythonConductor(BaseConductor):
# Move the contents of the execution directory to the final output
# directory.
job_output_dir = os.path.join(job[PYTHON_OUTPUT_DIR], job[JOB_ID])
job_output_dir = \
os.path.join(self.job_output_dir, os.path.basename(job_dir))
shutil.move(job_dir, job_output_dir)
def _is_valid_job_queue_dir(self, job_queue_dir)->None:
"""Validation check for 'job_queue_dir' variable from main
constructor."""
valid_dir_path(job_queue_dir, must_exist=False)
if not os.path.exists(job_queue_dir):
make_dir(job_queue_dir)
def _is_valid_job_output_dir(self, job_output_dir)->None:
"""Validation check for 'job_output_dir' variable from main
constructor."""
valid_dir_path(job_output_dir, must_exist=False)
if not os.path.exists(job_output_dir):
make_dir(job_output_dir)