refactored correctness dir away from core sub dir into either functionality or core as appropriate
This commit is contained in:
@ -14,13 +14,13 @@ from datetime import datetime
|
|||||||
from typing import Any, Dict, Tuple
|
from typing import Any, Dict, Tuple
|
||||||
|
|
||||||
from meow_base.core.base_conductor import BaseConductor
|
from meow_base.core.base_conductor import BaseConductor
|
||||||
from meow_base.core.correctness.meow import valid_job
|
from meow_base.core.meow import valid_job
|
||||||
from meow_base.core.correctness.vars import DEFAULT_JOB_QUEUE_DIR, \
|
from meow_base.core.vars import DEFAULT_JOB_QUEUE_DIR, \
|
||||||
DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, META_FILE, JOB_STATUS, \
|
DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, META_FILE, JOB_STATUS, \
|
||||||
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
|
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
|
||||||
JOB_ERROR, JOB_TYPE, DEFAULT_JOB_QUEUE_DIR, STATUS_RUNNING, \
|
JOB_ERROR, JOB_TYPE, DEFAULT_JOB_QUEUE_DIR, STATUS_RUNNING, \
|
||||||
JOB_START_TIME, DEFAULT_JOB_OUTPUT_DIR, get_job_file
|
JOB_START_TIME, DEFAULT_JOB_OUTPUT_DIR, get_job_file
|
||||||
from meow_base.core.correctness.validation import valid_dir_path
|
from meow_base.functionality.validation import valid_dir_path
|
||||||
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
|
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
|
||||||
write_yaml
|
write_yaml
|
||||||
|
|
||||||
|
@ -12,13 +12,13 @@ from datetime import datetime
|
|||||||
from typing import Any, Tuple, Dict
|
from typing import Any, Tuple, Dict
|
||||||
|
|
||||||
from meow_base.core.base_conductor import BaseConductor
|
from meow_base.core.base_conductor import BaseConductor
|
||||||
from meow_base.core.correctness.meow import valid_job
|
from meow_base.core.meow import valid_job
|
||||||
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, \
|
from meow_base.core.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, \
|
||||||
JOB_STATUS, STATUS_RUNNING, JOB_START_TIME, META_FILE, \
|
JOB_STATUS, STATUS_RUNNING, JOB_START_TIME, META_FILE, \
|
||||||
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
|
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
|
||||||
JOB_ERROR, JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, \
|
JOB_ERROR, JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, \
|
||||||
DEFAULT_JOB_OUTPUT_DIR
|
DEFAULT_JOB_OUTPUT_DIR
|
||||||
from meow_base.core.correctness.validation import valid_dir_path
|
from meow_base.functionality.validation import valid_dir_path
|
||||||
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
|
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
|
||||||
write_yaml
|
write_yaml
|
||||||
|
|
||||||
|
@ -8,9 +8,9 @@ Author(s): David Marchant
|
|||||||
|
|
||||||
from typing import Any, Tuple, Dict
|
from typing import Any, Tuple, Dict
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import VALID_CONDUCTOR_NAME_CHARS, \
|
from meow_base.core.vars import VALID_CONDUCTOR_NAME_CHARS, \
|
||||||
get_drt_imp_msg
|
get_drt_imp_msg
|
||||||
from meow_base.core.correctness.validation import check_implementation, \
|
from meow_base.functionality.validation import check_implementation, \
|
||||||
valid_string
|
valid_string
|
||||||
from meow_base.functionality.naming import generate_conductor_id
|
from meow_base.functionality.naming import generate_conductor_id
|
||||||
|
|
||||||
|
@ -8,9 +8,9 @@ Author(s): David Marchant
|
|||||||
|
|
||||||
from typing import Any, Tuple, Dict
|
from typing import Any, Tuple, Dict
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import VALID_CHANNELS, \
|
from meow_base.core.vars import VALID_CHANNELS, \
|
||||||
VALID_HANDLER_NAME_CHARS, get_drt_imp_msg
|
VALID_HANDLER_NAME_CHARS, get_drt_imp_msg
|
||||||
from meow_base.core.correctness.validation import check_implementation, \
|
from meow_base.functionality.validation import check_implementation, \
|
||||||
valid_string
|
valid_string
|
||||||
from meow_base.functionality.naming import generate_handler_id
|
from meow_base.functionality.naming import generate_handler_id
|
||||||
|
|
||||||
|
@ -12,9 +12,9 @@ from typing import Union, Dict
|
|||||||
from meow_base.core.base_pattern import BasePattern
|
from meow_base.core.base_pattern import BasePattern
|
||||||
from meow_base.core.base_recipe import BaseRecipe
|
from meow_base.core.base_recipe import BaseRecipe
|
||||||
from meow_base.core.rule import Rule
|
from meow_base.core.rule import Rule
|
||||||
from meow_base.core.correctness.vars import VALID_CHANNELS, \
|
from meow_base.core.vars import VALID_CHANNELS, \
|
||||||
VALID_MONITOR_NAME_CHARS, get_drt_imp_msg
|
VALID_MONITOR_NAME_CHARS, get_drt_imp_msg
|
||||||
from meow_base.core.correctness.validation import check_implementation, \
|
from meow_base.functionality.validation import check_implementation, \
|
||||||
valid_string
|
valid_string
|
||||||
from meow_base.functionality.meow import create_rules
|
from meow_base.functionality.meow import create_rules
|
||||||
from meow_base.functionality.naming import generate_monitor_id
|
from meow_base.functionality.naming import generate_monitor_id
|
||||||
|
@ -10,9 +10,9 @@ import itertools
|
|||||||
|
|
||||||
from typing import Any, Union, Tuple, Dict, List
|
from typing import Any, Union, Tuple, Dict, List
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import VALID_PATTERN_NAME_CHARS, \
|
from meow_base.core.vars import VALID_PATTERN_NAME_CHARS, \
|
||||||
SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
|
SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
|
||||||
from meow_base.core.correctness.validation import valid_string, check_type, \
|
from meow_base.functionality.validation import valid_string, check_type, \
|
||||||
check_implementation, valid_dict
|
check_implementation, valid_dict
|
||||||
|
|
||||||
|
|
||||||
|
@ -8,9 +8,9 @@ Author(s): David Marchant
|
|||||||
|
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
|
from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \
|
||||||
get_drt_imp_msg
|
get_drt_imp_msg
|
||||||
from meow_base.core.correctness.validation import check_implementation, \
|
from meow_base.functionality.validation import check_implementation, \
|
||||||
valid_string
|
valid_string
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,10 +1,16 @@
|
|||||||
|
|
||||||
|
"""
|
||||||
|
This file contains certain meow specific defintions, most notably the
|
||||||
|
dictionaries passed around within the runner.
|
||||||
|
|
||||||
|
Author(s): David Marchant
|
||||||
|
"""
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, Type
|
from typing import Any, Dict, Type
|
||||||
|
|
||||||
from meow_base.core.rule import Rule
|
from meow_base.core.rule import Rule
|
||||||
from meow_base.core.correctness.validation import check_type
|
from meow_base.functionality.validation import check_type
|
||||||
from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \
|
from meow_base.core.vars import EVENT_TYPE, EVENT_PATH, \
|
||||||
JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
|
JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
|
||||||
JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
|
JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
|
||||||
|
|
||||||
@ -12,7 +18,6 @@ from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \
|
|||||||
EVENT_KEYS = {
|
EVENT_KEYS = {
|
||||||
EVENT_TYPE: str,
|
EVENT_TYPE: str,
|
||||||
EVENT_PATH: str,
|
EVENT_PATH: str,
|
||||||
# Should be a Rule but can't import here due to circular dependencies
|
|
||||||
EVENT_RULE: Rule
|
EVENT_RULE: Rule
|
||||||
}
|
}
|
||||||
|
|
@ -12,9 +12,9 @@ if "BasePattern" not in modules:
|
|||||||
from meow_base.core.base_pattern import BasePattern
|
from meow_base.core.base_pattern import BasePattern
|
||||||
if "BaseRecipe" not in modules:
|
if "BaseRecipe" not in modules:
|
||||||
from meow_base.core.base_recipe import BaseRecipe
|
from meow_base.core.base_recipe import BaseRecipe
|
||||||
from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \
|
from meow_base.core.vars import VALID_RULE_NAME_CHARS, \
|
||||||
get_drt_imp_msg
|
get_drt_imp_msg
|
||||||
from meow_base.core.correctness.validation import valid_string, check_type, \
|
from meow_base.functionality.validation import valid_string, check_type, \
|
||||||
check_implementation
|
check_implementation
|
||||||
from meow_base.functionality.naming import generate_rule_id
|
from meow_base.functionality.naming import generate_rule_id
|
||||||
|
|
||||||
|
@ -17,10 +17,10 @@ from typing import Any, Union, Dict, List, Type
|
|||||||
from meow_base.core.base_conductor import BaseConductor
|
from meow_base.core.base_conductor import BaseConductor
|
||||||
from meow_base.core.base_handler import BaseHandler
|
from meow_base.core.base_handler import BaseHandler
|
||||||
from meow_base.core.base_monitor import BaseMonitor
|
from meow_base.core.base_monitor import BaseMonitor
|
||||||
from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, \
|
from meow_base.core.vars import DEBUG_WARNING, DEBUG_INFO, \
|
||||||
EVENT_TYPE, VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, \
|
EVENT_TYPE, VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, \
|
||||||
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH
|
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH
|
||||||
from meow_base.core.correctness.validation import check_type, valid_list, \
|
from meow_base.functionality.validation import check_type, valid_list, \
|
||||||
valid_dir_path
|
valid_dir_path
|
||||||
from meow_base.functionality.debug import setup_debugging, print_debug
|
from meow_base.functionality.debug import setup_debugging, print_debug
|
||||||
from meow_base.functionality.file_io import make_dir, read_yaml
|
from meow_base.functionality.file_io import make_dir, read_yaml
|
||||||
|
@ -6,8 +6,8 @@ Author(s): David Marchant
|
|||||||
|
|
||||||
from typing import Any, Tuple
|
from typing import Any, Tuple
|
||||||
|
|
||||||
from meow_base.core.correctness.validation import check_type
|
from meow_base.functionality.validation import check_type
|
||||||
from meow_base.core.correctness.vars import DEBUG_INFO, DEBUG_WARNING
|
from meow_base.core.vars import DEBUG_INFO, DEBUG_WARNING
|
||||||
|
|
||||||
|
|
||||||
def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:
|
def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:
|
||||||
|
@ -11,7 +11,7 @@ from os import makedirs, remove, rmdir, walk
|
|||||||
from os.path import exists, isfile, join
|
from os.path import exists, isfile, join
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
from meow_base.core.correctness.validation import valid_path
|
from meow_base.functionality.validation import valid_path
|
||||||
|
|
||||||
|
|
||||||
def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):
|
def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):
|
||||||
|
@ -9,8 +9,8 @@ from os import listdir
|
|||||||
from os.path import isfile
|
from os.path import isfile
|
||||||
|
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import HASH_BUFFER_SIZE, SHA256
|
from meow_base.core.vars import HASH_BUFFER_SIZE, SHA256
|
||||||
from meow_base.core.correctness.validation import check_type, \
|
from meow_base.functionality.validation import check_type, \
|
||||||
valid_existing_file_path, valid_existing_dir_path
|
valid_existing_file_path, valid_existing_dir_path
|
||||||
|
|
||||||
def _get_file_sha256(file_path:str)->str:
|
def _get_file_sha256(file_path:str)->str:
|
||||||
|
@ -11,9 +11,9 @@ from typing import Any, Dict, Union, List
|
|||||||
from meow_base.core.base_pattern import BasePattern
|
from meow_base.core.base_pattern import BasePattern
|
||||||
from meow_base.core.base_recipe import BaseRecipe
|
from meow_base.core.base_recipe import BaseRecipe
|
||||||
from meow_base.core.rule import Rule
|
from meow_base.core.rule import Rule
|
||||||
from meow_base.core.correctness.validation import check_type, valid_dict, \
|
from meow_base.functionality.validation import check_type, valid_dict, \
|
||||||
valid_list
|
valid_list
|
||||||
from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \
|
from meow_base.core.vars import EVENT_PATH, EVENT_RULE, \
|
||||||
EVENT_TYPE, EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \
|
EVENT_TYPE, EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \
|
||||||
JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
|
JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
|
||||||
JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \
|
JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \
|
||||||
|
@ -7,7 +7,7 @@ Author(s): David Marchant
|
|||||||
from typing import List
|
from typing import List
|
||||||
from random import SystemRandom
|
from random import SystemRandom
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
|
from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
|
||||||
|
|
||||||
|
|
||||||
#TODO Make this guaranteed unique
|
#TODO Make this guaranteed unique
|
||||||
|
@ -10,7 +10,7 @@ from os import getenv
|
|||||||
from papermill.translators import papermill_translators
|
from papermill.translators import papermill_translators
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
from meow_base.core.correctness.validation import check_script, check_type
|
from meow_base.functionality.validation import check_script, check_type
|
||||||
|
|
||||||
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
|
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
|
||||||
def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any],
|
def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any],
|
||||||
|
@ -12,7 +12,7 @@ from multiprocessing.connection import Connection, wait as multi_wait
|
|||||||
if osName == 'nt':
|
if osName == 'nt':
|
||||||
from multiprocessing.connection import PipeConnection
|
from multiprocessing.connection import PipeConnection
|
||||||
from multiprocessing.queues import Queue
|
from multiprocessing.queues import Queue
|
||||||
from meow_base.core.correctness.vars import VALID_CHANNELS
|
from meow_base.core.vars import VALID_CHANNELS
|
||||||
|
|
||||||
|
|
||||||
def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:
|
def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:
|
||||||
|
@ -11,7 +11,7 @@ from os.path import basename
|
|||||||
from sys import version_info, prefix, base_prefix
|
from sys import version_info, prefix, base_prefix
|
||||||
from typing import Any, Dict, List, Tuple, Union
|
from typing import Any, Dict, List, Tuple, Union
|
||||||
|
|
||||||
from meow_base.core.correctness.validation import check_type
|
from meow_base.functionality.validation import check_type
|
||||||
|
|
||||||
REQUIREMENT_PYTHON = "python"
|
REQUIREMENT_PYTHON = "python"
|
||||||
REQ_PYTHON_MODULES = "modules"
|
REQ_PYTHON_MODULES = "modules"
|
||||||
|
@ -11,7 +11,7 @@ from os.path import sep, exists, isfile, isdir, dirname
|
|||||||
from typing import Any, _SpecialForm, Union, Type, Dict, List, \
|
from typing import Any, _SpecialForm, Union, Type, Dict, List, \
|
||||||
get_origin, get_args
|
get_origin, get_args
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
|
from meow_base.core.vars import VALID_PATH_CHARS, get_not_imp_msg
|
||||||
|
|
||||||
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
|
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
|
||||||
or_none:bool=False, hint:str="")->None:
|
or_none:bool=False, hint:str="")->None:
|
@ -22,9 +22,9 @@ from meow_base.core.base_recipe import BaseRecipe
|
|||||||
from meow_base.core.base_monitor import BaseMonitor
|
from meow_base.core.base_monitor import BaseMonitor
|
||||||
from meow_base.core.base_pattern import BasePattern
|
from meow_base.core.base_pattern import BasePattern
|
||||||
from meow_base.core.rule import Rule
|
from meow_base.core.rule import Rule
|
||||||
from meow_base.core.correctness.validation import check_type, valid_string, \
|
from meow_base.functionality.validation import check_type, valid_string, \
|
||||||
valid_dict, valid_list, valid_dir_path
|
valid_dict, valid_list, valid_dir_path
|
||||||
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
|
from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \
|
||||||
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
|
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
|
||||||
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, DIR_EVENTS, \
|
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, DIR_EVENTS, \
|
||||||
FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT, \
|
FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT, \
|
||||||
|
@ -7,10 +7,10 @@ from typing import Any, Dict, List, Tuple
|
|||||||
|
|
||||||
from meow_base.core.base_handler import BaseHandler
|
from meow_base.core.base_handler import BaseHandler
|
||||||
from meow_base.core.base_recipe import BaseRecipe
|
from meow_base.core.base_recipe import BaseRecipe
|
||||||
from meow_base.core.correctness.meow import valid_event
|
from meow_base.core.meow import valid_event
|
||||||
from meow_base.core.correctness.validation import check_type, valid_dict, \
|
from meow_base.functionality.validation import check_type, valid_dict, \
|
||||||
valid_string, valid_dir_path
|
valid_string, valid_dir_path
|
||||||
from meow_base.core.correctness.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
|
from meow_base.core.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
|
||||||
VALID_VARIABLE_NAME_CHARS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, JOB_ID, \
|
VALID_VARIABLE_NAME_CHARS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, JOB_ID, \
|
||||||
EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_PARAMETERS, WATCHDOG_HASH, \
|
EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_PARAMETERS, WATCHDOG_HASH, \
|
||||||
WATCHDOG_BASE, META_FILE, STATUS_QUEUED, JOB_STATUS, \
|
WATCHDOG_BASE, META_FILE, STATUS_QUEUED, JOB_STATUS, \
|
||||||
|
@ -13,10 +13,10 @@ from typing import Any, Tuple, Dict
|
|||||||
|
|
||||||
from meow_base.core.base_recipe import BaseRecipe
|
from meow_base.core.base_recipe import BaseRecipe
|
||||||
from meow_base.core.base_handler import BaseHandler
|
from meow_base.core.base_handler import BaseHandler
|
||||||
from meow_base.core.correctness.meow import valid_event
|
from meow_base.core.meow import valid_event
|
||||||
from meow_base.core.correctness.validation import check_type, valid_string, \
|
from meow_base.functionality.validation import check_type, valid_string, \
|
||||||
valid_dict, valid_path, valid_dir_path, valid_existing_file_path
|
valid_dict, valid_path, valid_dir_path, valid_existing_file_path
|
||||||
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \
|
from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
|
||||||
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
|
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
|
||||||
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, \
|
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, \
|
||||||
JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, PARAMS_FILE, \
|
JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, PARAMS_FILE, \
|
||||||
@ -203,7 +203,7 @@ def papermill_job_func(job_dir):
|
|||||||
import os
|
import os
|
||||||
import papermill
|
import papermill
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
|
from meow_base.core.vars import JOB_EVENT, JOB_ID, \
|
||||||
EVENT_PATH, META_FILE, PARAMS_FILE, \
|
EVENT_PATH, META_FILE, PARAMS_FILE, \
|
||||||
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
|
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
|
||||||
JOB_ERROR, STATUS_FAILED, get_job_file, \
|
JOB_ERROR, STATUS_FAILED, get_job_file, \
|
||||||
|
@ -12,10 +12,10 @@ from typing import Any, Tuple, Dict, List
|
|||||||
|
|
||||||
from meow_base.core.base_recipe import BaseRecipe
|
from meow_base.core.base_recipe import BaseRecipe
|
||||||
from meow_base.core.base_handler import BaseHandler
|
from meow_base.core.base_handler import BaseHandler
|
||||||
from meow_base.core.correctness.meow import valid_event
|
from meow_base.core.meow import valid_event
|
||||||
from meow_base.core.correctness.validation import check_script, valid_string, \
|
from meow_base.functionality.validation import check_script, valid_string, \
|
||||||
valid_dict, valid_dir_path
|
valid_dict, valid_dir_path
|
||||||
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \
|
from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
|
||||||
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
|
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
|
||||||
DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, \
|
DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, \
|
||||||
WATCHDOG_HASH, JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, \
|
WATCHDOG_HASH, JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, \
|
||||||
@ -179,7 +179,7 @@ def python_job_func(job_dir):
|
|||||||
import os
|
import os
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
|
from meow_base.core.vars import JOB_EVENT, JOB_ID, \
|
||||||
EVENT_PATH, META_FILE, PARAMS_FILE, \
|
EVENT_PATH, META_FILE, PARAMS_FILE, \
|
||||||
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
|
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
|
||||||
JOB_ERROR, STATUS_FAILED, get_base_file, \
|
JOB_ERROR, STATUS_FAILED, get_base_file, \
|
||||||
|
@ -7,7 +7,7 @@ import os
|
|||||||
|
|
||||||
from distutils.dir_util import copy_tree
|
from distutils.dir_util import copy_tree
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \
|
from meow_base.core.vars import DEFAULT_JOB_OUTPUT_DIR, \
|
||||||
DEFAULT_JOB_QUEUE_DIR
|
DEFAULT_JOB_QUEUE_DIR
|
||||||
from meow_base.functionality.file_io import make_dir, rmtree
|
from meow_base.functionality.file_io import make_dir, rmtree
|
||||||
from meow_base.patterns.file_event_pattern import FileEventPattern
|
from meow_base.patterns.file_event_pattern import FileEventPattern
|
||||||
|
@ -11,10 +11,8 @@ skip_long_tests=0
|
|||||||
|
|
||||||
for arg in "$@"
|
for arg in "$@"
|
||||||
do
|
do
|
||||||
echo "Given arg: $arg";
|
|
||||||
if [[ $arg == -s ]]
|
if [[ $arg == -s ]]
|
||||||
then
|
then
|
||||||
echo "skippy the kangaroo"
|
|
||||||
skip_long_tests=1
|
skip_long_tests=1
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
@ -31,7 +29,6 @@ do
|
|||||||
&& [[ $entry != ./shared.py ]];
|
&& [[ $entry != ./shared.py ]];
|
||||||
then
|
then
|
||||||
SKIP_LONG=$skip_long_tests pytest $entry "-W ignore::DeprecationWarning"
|
SKIP_LONG=$skip_long_tests pytest $entry "-W ignore::DeprecationWarning"
|
||||||
# SKIP_LONG=$skip_long_tests pytest $entry
|
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
@ -8,7 +8,7 @@ from meow_base.core.base_handler import BaseHandler
|
|||||||
from meow_base.core.base_monitor import BaseMonitor
|
from meow_base.core.base_monitor import BaseMonitor
|
||||||
from meow_base.core.base_pattern import BasePattern
|
from meow_base.core.base_pattern import BasePattern
|
||||||
from meow_base.core.base_recipe import BaseRecipe
|
from meow_base.core.base_recipe import BaseRecipe
|
||||||
from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
|
from meow_base.core.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
|
||||||
from meow_base.patterns.file_event_pattern import FileEventPattern
|
from meow_base.patterns.file_event_pattern import FileEventPattern
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown
|
||||||
|
|
||||||
|
@ -6,7 +6,7 @@ import unittest
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, \
|
from meow_base.core.vars import JOB_TYPE_PYTHON, SHA256, \
|
||||||
JOB_PARAMETERS, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, \
|
JOB_PARAMETERS, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, \
|
||||||
JOB_EVENT, META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \
|
JOB_EVENT, META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \
|
||||||
JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \
|
JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \
|
||||||
|
@ -12,7 +12,7 @@ from time import sleep
|
|||||||
from typing import Dict
|
from typing import Dict
|
||||||
|
|
||||||
from meow_base.core.rule import Rule
|
from meow_base.core.rule import Rule
|
||||||
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
||||||
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
|
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
|
||||||
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, \
|
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, \
|
||||||
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
|
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
|
||||||
|
@ -6,7 +6,7 @@ import unittest
|
|||||||
from multiprocessing import Pipe
|
from multiprocessing import Pipe
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
|
from meow_base.core.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
|
||||||
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
|
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
|
||||||
SWEEP_JUMP, SWEEP_STOP, DIR_EVENTS
|
SWEEP_JUMP, SWEEP_STOP, DIR_EVENTS
|
||||||
from meow_base.functionality.file_io import make_dir
|
from meow_base.functionality.file_io import make_dir
|
||||||
|
@ -8,8 +8,8 @@ import unittest
|
|||||||
from multiprocessing import Pipe
|
from multiprocessing import Pipe
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
|
|
||||||
from meow_base.core.correctness.meow import valid_job
|
from meow_base.core.meow import valid_job
|
||||||
from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, \
|
from meow_base.core.vars import EVENT_TYPE, WATCHDOG_BASE, \
|
||||||
EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \
|
EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \
|
||||||
JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, PYTHON_FUNC, \
|
JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, PYTHON_FUNC, \
|
||||||
JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \
|
JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \
|
||||||
|
@ -13,7 +13,7 @@ from meow_base.core.base_conductor import BaseConductor
|
|||||||
from meow_base.core.base_handler import BaseHandler
|
from meow_base.core.base_handler import BaseHandler
|
||||||
from meow_base.core.base_monitor import BaseMonitor
|
from meow_base.core.base_monitor import BaseMonitor
|
||||||
from meow_base.conductors import LocalPythonConductor
|
from meow_base.conductors import LocalPythonConductor
|
||||||
from meow_base.core.correctness.vars import JOB_TYPE_PAPERMILL, JOB_ERROR, \
|
from meow_base.core.vars import JOB_TYPE_PAPERMILL, JOB_ERROR, \
|
||||||
META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME, get_result_file
|
META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME, get_result_file
|
||||||
from meow_base.core.runner import MeowRunner
|
from meow_base.core.runner import MeowRunner
|
||||||
from meow_base.functionality.file_io import make_dir, read_file, \
|
from meow_base.functionality.file_io import make_dir, read_file, \
|
||||||
@ -972,7 +972,7 @@ class MeowTests(unittest.TestCase):
|
|||||||
|
|
||||||
# Test some actual scientific analysis, but in a simple progression
|
# Test some actual scientific analysis, but in a simple progression
|
||||||
def testScientificAnalysisAllGood(self)->None:
|
def testScientificAnalysisAllGood(self)->None:
|
||||||
if os.environ["SKIP_LONG"]:
|
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
|
||||||
warn("Skipping testScientificAnalysisAllGood")
|
warn("Skipping testScientificAnalysisAllGood")
|
||||||
return
|
return
|
||||||
|
|
||||||
@ -1096,7 +1096,7 @@ class MeowTests(unittest.TestCase):
|
|||||||
|
|
||||||
# Test some actual scientific analysis, in a predicatable loop
|
# Test some actual scientific analysis, in a predicatable loop
|
||||||
def testScientificAnalysisPredictableLoop(self)->None:
|
def testScientificAnalysisPredictableLoop(self)->None:
|
||||||
if os.environ["SKIP_LONG"]:
|
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
|
||||||
warn("Skipping testScientificAnalysisPredictableLoop")
|
warn("Skipping testScientificAnalysisPredictableLoop")
|
||||||
return
|
return
|
||||||
|
|
||||||
@ -1235,7 +1235,7 @@ class MeowTests(unittest.TestCase):
|
|||||||
|
|
||||||
# Test some actual scientific analysis, in an unpredicatable loop
|
# Test some actual scientific analysis, in an unpredicatable loop
|
||||||
def testScientificAnalysisRandomLoop(self)->None:
|
def testScientificAnalysisRandomLoop(self)->None:
|
||||||
if os.environ["SKIP_LONG"]:
|
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
|
||||||
warn("Skipping testScientificAnalysisRandomLoop")
|
warn("Skipping testScientificAnalysisRandomLoop")
|
||||||
return
|
return
|
||||||
|
|
||||||
@ -1387,7 +1387,7 @@ class MeowTests(unittest.TestCase):
|
|||||||
|
|
||||||
# Test some actual scientific analysis, in an unpredicatable loop
|
# Test some actual scientific analysis, in an unpredicatable loop
|
||||||
def testScientificAnalysisMassiveRandomLoop(self)->None:
|
def testScientificAnalysisMassiveRandomLoop(self)->None:
|
||||||
if os.environ["SKIP_LONG"]:
|
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
|
||||||
warn("Skipping testScientificAnalysisMassiveRandomLoop")
|
warn("Skipping testScientificAnalysisMassiveRandomLoop")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -5,13 +5,13 @@ import os
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Union
|
from typing import Any, Union
|
||||||
|
|
||||||
from meow_base.core.correctness.meow import valid_event, valid_job, \
|
from meow_base.core.meow import valid_event, valid_job, \
|
||||||
valid_watchdog_event
|
valid_watchdog_event
|
||||||
from meow_base.core.correctness.validation import check_type, \
|
from meow_base.functionality.validation import check_type, \
|
||||||
check_implementation, valid_string, valid_dict, valid_list, \
|
check_implementation, valid_string, valid_dict, valid_list, \
|
||||||
valid_existing_file_path, valid_dir_path, valid_non_existing_path, \
|
valid_existing_file_path, valid_dir_path, valid_non_existing_path, \
|
||||||
check_callable
|
check_callable
|
||||||
from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, \
|
from meow_base.core.vars import VALID_NAME_CHARS, SHA256, \
|
||||||
EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
|
EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
|
||||||
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
|
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
|
||||||
WATCHDOG_BASE, WATCHDOG_HASH
|
WATCHDOG_BASE, WATCHDOG_HASH
|
||||||
|
Reference in New Issue
Block a user