NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformatted imports to work better on other machines, plus added benchmarking to project

Browse Source
This commit is contained in:
PatchOfScotland
2023-03-13 11:32:45 +01:00
parent c01df1b190
commit 40ed98000b
48 changed files with 907 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
recipes/__init__.py
View File

@ -1,4 +1,4 @@
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler
from recipes.python_recipe import PythonRecipe, PythonHandler
from .jupyter_notebook_recipe import JupyterNotebookRecipe, PapermillHandler, \
getRecipeFromNotebook
from .python_recipe import PythonRecipe, PythonHandler

43
recipes/jupyter_notebook_recipe.py
View File

@ -11,21 +11,21 @@ import sys
from typing import Any, Tuple, Dict
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.meow import valid_event
from core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path, valid_existing_file_path
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, JOB_PARAMETERS, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_RULE, EVENT_TYPE, \
EVENT_RULE, get_base_file
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_notebook, write_notebook, \
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_notebook, write_notebook, \
write_yaml
from functionality.meow import create_job, replace_keywords
from meow_base.functionality.meow import create_job, replace_keywords
class JupyterNotebookRecipe(BaseRecipe):
@ -182,20 +182,37 @@ class PapermillHandler(BaseHandler):
# Send job directory, as actual definitons will be read from within it
self.to_runner.send(job_dir)
#TODO test me
def getRecipeFromNotebook(name:str, notebook_filename:str,
parameters:Dict[str,Any]={}, requirements:Dict[str,Any]={}
)->JupyterNotebookRecipe:
valid_existing_file_path(notebook_filename, extension=".ipynb")
check_type(name, str, hint="getRecipeFromNotebook.name")
notebook_code = read_notebook(notebook_filename)
return JupyterNotebookRecipe(
name,
notebook_code,
parameters=parameters,
requirements=requirements,
source=notebook_filename
)
# Papermill job execution code, to be run within the conductor
def papermill_job_func(job_dir):
# Requires own imports as will be run in its own execution environment
import os
import papermill
from datetime import datetime
from core.correctness.vars import JOB_EVENT, JOB_ID, \
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_job_file, \
get_result_file
from functionality.file_io import read_yaml, write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.parameterisation import parameterize_jupyter_notebook
from meow_base.functionality.file_io import read_yaml, write_notebook, write_yaml
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.parameterisation import parameterize_jupyter_notebook
# Identify job files

24
recipes/python_recipe.py
View File

@ -10,21 +10,21 @@ import sys
from typing import Any, Tuple, Dict, List
from core.base_recipe import BaseRecipe
from core.base_handler import BaseHandler
from core.correctness.meow import valid_event
from core.correctness.validation import check_script, valid_string, \
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_script, valid_string, \
valid_dict, valid_dir_path
from core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, PYTHON_FUNC, \
DEBUG_INFO, EVENT_TYPE_WATCHDOG, JOB_HASH, DEFAULT_JOB_QUEUE_DIR, \
EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, WATCHDOG_HASH, JOB_PARAMETERS, \
JOB_ID, WATCHDOG_BASE, META_FILE, \
PARAMS_FILE, JOB_STATUS, STATUS_QUEUED, EVENT_TYPE, EVENT_RULE, \
get_base_file
from functionality.debug import setup_debugging, print_debug
from functionality.file_io import make_dir, read_file_lines, write_file, \
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_file_lines, write_file, \
write_yaml, lines_to_string
from functionality.meow import create_job, replace_keywords
from meow_base.functionality.meow import create_job, replace_keywords
class PythonRecipe(BaseRecipe):
@ -180,14 +180,14 @@ def python_job_func(job_dir):
import os
from datetime import datetime
from io import StringIO
from core.correctness.vars import JOB_EVENT, JOB_ID, \
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, JOB_HASH, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_base_file, \
get_job_file, get_result_file
from functionality.file_io import read_yaml, write_yaml
from functionality.hashing import get_file_hash
from functionality.parameterisation import parameterize_python_script
from meow_base.functionality.file_io import read_yaml, write_yaml
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.parameterisation import parameterize_python_script
# Identify job files
meta_file = os.path.join(job_dir, META_FILE)