NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added standardised job creation

Browse Source
This commit is contained in:
PatchOfScotland
2023-04-22 21:48:33 +02:00
parent f306d8b6f2
commit d3eb2dbf9f
15 changed files with 515 additions and 1281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
conductors/local_bash_conductor.py
View File

@ -7,22 +7,16 @@ Author(s): David Marchant
"""
import os
import shutil
import subprocess
from datetime import datetime
from typing import Any, Dict, Tuple
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.meow import valid_job
from meow_base.core.vars import DEFAULT_JOB_QUEUE_DIR, \
DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, META_FILE, JOB_STATUS, \
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
JOB_ERROR, JOB_TYPE, DEFAULT_JOB_QUEUE_DIR, STATUS_RUNNING, \
JOB_START_TIME, DEFAULT_JOB_OUTPUT_DIR, get_job_file
DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, JOB_TYPE, \
DEFAULT_JOB_QUEUE_DIR, DEFAULT_JOB_OUTPUT_DIR
from meow_base.functionality.validation import valid_dir_path
from meow_base.functionality.file_io import make_dir, write_file, \
threadsafe_read_status, threadsafe_update_status
from meow_base.functionality.file_io import make_dir
class LocalBashConductor(BaseConductor):
@ -54,90 +48,6 @@ class LocalBashConductor(BaseConductor):
except Exception as e:
return False, str(e)
def execute(self, job_dir:str)->None:
"""Function to actually execute a Bash job. This will read job
defintions from its meta file, update the meta file and attempt to
execute. Some unspecific feedback will be given on execution failure,
but depending on what it is it may be up to the job itself to provide
more detailed feedback."""
valid_dir_path(job_dir, must_exist=True)
# Test our job parameters. Even if its gibberish, we still move to
# output
abort = False
try:
meta_file = os.path.join(job_dir, META_FILE)
job = threadsafe_read_status(meta_file)
valid_job(job)
# update the status file with running status
threadsafe_update_status(
{
JOB_STATUS: STATUS_RUNNING,
JOB_START_TIME: datetime.now()
},
meta_file
)
except Exception as e:
# If something has gone wrong at this stage then its bad, so we
# need to make our own error file
error_file = os.path.join(job_dir, BACKUP_JOB_ERROR_FILE)
write_file(f"Recieved incorrectly setup job.\n\n{e}", error_file)
abort = True
# execute the job
if not abort:
try:
print(f"PWD: {os.getcwd()}")
print(f"job_dir: {job_dir}")
print(os.path.exists(os.path.join(job_dir, get_job_file(JOB_TYPE_BASH))))
result = subprocess.call(
os.path.join(job_dir, get_job_file(JOB_TYPE_BASH)),
cwd="."
)
if result == 0:
# Update the status file with the finalised status
threadsafe_update_status(
{
JOB_STATUS: STATUS_DONE,
JOB_END_TIME: datetime.now()
},
meta_file
)
else:
# Update the status file with the error status. Don't
# overwrite any more specific error messages already
# created
threadsafe_update_status(
{
JOB_STATUS: STATUS_FAILED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: "Job execution returned non-zero."
},
meta_file
)
except Exception as e:
# Update the status file with the error status. Don't overwrite
# any more specific error messages already created
threadsafe_update_status(
{
JOB_STATUS: STATUS_FAILED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: f"Job execution failed. {e}"
},
meta_file
)
# Move the contents of the execution directory to the final output
# directory.
job_output_dir = \
os.path.join(self.job_output_dir, os.path.basename(job_dir))
shutil.move(job_dir, job_output_dir)
def _is_valid_job_queue_dir(self, job_queue_dir)->None:
"""Validation check for 'job_queue_dir' variable from main
constructor."""

66
conductors/local_python_conductor.py
View File

@ -51,72 +51,6 @@ class LocalPythonConductor(BaseConductor):
except Exception as e:
return False, str(e)
def execute(self, job_dir:str)->None:
"""Function to actually execute a Python job. This will read job
defintions from its meta file, update the meta file and attempt to
execute. Some unspecific feedback will be given on execution failure,
but depending on what it is it may be up to the job itself to provide
more detailed feedback."""
valid_dir_path(job_dir, must_exist=True)
# Test our job parameters. Even if its gibberish, we still move to
# output
abort = False
try:
meta_file = os.path.join(job_dir, META_FILE)
job = threadsafe_read_status(meta_file)
valid_job(job)
# update the status file with running status
threadsafe_update_status(
{
JOB_STATUS: STATUS_RUNNING,
JOB_START_TIME: datetime.now()
},
meta_file
)
except Exception as e:
# If something has gone wrong at this stage then its bad, so we
# need to make our own error file
error_file = os.path.join(job_dir, BACKUP_JOB_ERROR_FILE)
write_file(f"Recieved incorrectly setup job.\n\n{e}", error_file)
abort = True
# execute the job
if not abort:
try:
job_function = job[PYTHON_FUNC]
job_function(job_dir)
# Update the status file with the finalised status
threadsafe_update_status(
{
JOB_STATUS: STATUS_DONE,
JOB_END_TIME: datetime.now()
},
meta_file
)
except Exception as e:
# Update the status file with the error status. Don't overwrite
# any more specific error messages already created
threadsafe_update_status(
{
JOB_STATUS: STATUS_FAILED,
JOB_END_TIME: datetime.now(),
JOB_ERROR: f"Job execution failed. {e}"
},
meta_file
)
# Move the contents of the execution directory to the final output
# directory.
job_output_dir = \
os.path.join(self.job_output_dir, os.path.basename(job_dir))
shutil.move(job_dir, job_output_dir)
def _is_valid_job_queue_dir(self, job_queue_dir)->None:
"""Validation check for 'job_queue_dir' variable from main
constructor."""