refactored correctness dir away from core sub dir into either functionality or core as appropriate

This commit is contained in:
PatchOfScotland
2023-03-31 15:55:16 +02:00
parent 5952b02be4
commit 547d5fefce
35 changed files with 69 additions and 67 deletions

View File

@ -14,13 +14,13 @@ from datetime import datetime
from typing import Any, Dict, Tuple
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import DEFAULT_JOB_QUEUE_DIR, \
from meow_base.core.meow import valid_job
from meow_base.core.vars import DEFAULT_JOB_QUEUE_DIR, \
DEFAULT_JOB_OUTPUT_DIR, JOB_TYPE, JOB_TYPE_BASH, META_FILE, JOB_STATUS, \
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
JOB_ERROR, JOB_TYPE, DEFAULT_JOB_QUEUE_DIR, STATUS_RUNNING, \
JOB_START_TIME, DEFAULT_JOB_OUTPUT_DIR, get_job_file
from meow_base.core.correctness.validation import valid_dir_path
from meow_base.functionality.validation import valid_dir_path
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
write_yaml

View File

@ -12,13 +12,13 @@ from datetime import datetime
from typing import Any, Tuple, Dict
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, \
from meow_base.core.meow import valid_job
from meow_base.core.vars import JOB_TYPE_PYTHON, PYTHON_FUNC, \
JOB_STATUS, STATUS_RUNNING, JOB_START_TIME, META_FILE, \
BACKUP_JOB_ERROR_FILE, STATUS_DONE, JOB_END_TIME, STATUS_FAILED, \
JOB_ERROR, JOB_TYPE, JOB_TYPE_PAPERMILL, DEFAULT_JOB_QUEUE_DIR, \
DEFAULT_JOB_OUTPUT_DIR
from meow_base.core.correctness.validation import valid_dir_path
from meow_base.functionality.validation import valid_dir_path
from meow_base.functionality.file_io import make_dir, read_yaml, write_file, \
write_yaml

View File

View File

@ -8,9 +8,9 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from meow_base.core.correctness.vars import VALID_CONDUCTOR_NAME_CHARS, \
from meow_base.core.vars import VALID_CONDUCTOR_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
from meow_base.functionality.validation import check_implementation, \
valid_string
from meow_base.functionality.naming import generate_conductor_id

View File

@ -8,9 +8,9 @@ Author(s): David Marchant
from typing import Any, Tuple, Dict
from meow_base.core.correctness.vars import VALID_CHANNELS, \
from meow_base.core.vars import VALID_CHANNELS, \
VALID_HANDLER_NAME_CHARS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
from meow_base.functionality.validation import check_implementation, \
valid_string
from meow_base.functionality.naming import generate_handler_id

View File

@ -12,9 +12,9 @@ from typing import Union, Dict
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.rule import Rule
from meow_base.core.correctness.vars import VALID_CHANNELS, \
from meow_base.core.vars import VALID_CHANNELS, \
VALID_MONITOR_NAME_CHARS, get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
from meow_base.functionality.validation import check_implementation, \
valid_string
from meow_base.functionality.meow import create_rules
from meow_base.functionality.naming import generate_monitor_id

View File

@ -10,9 +10,9 @@ import itertools
from typing import Any, Union, Tuple, Dict, List
from meow_base.core.correctness.vars import VALID_PATTERN_NAME_CHARS, \
from meow_base.core.vars import VALID_PATTERN_NAME_CHARS, \
SWEEP_JUMP, SWEEP_START, SWEEP_STOP, get_drt_imp_msg
from meow_base.core.correctness.validation import valid_string, check_type, \
from meow_base.functionality.validation import valid_string, check_type, \
check_implementation, valid_dict

View File

@ -8,9 +8,9 @@ Author(s): David Marchant
from typing import Any, Dict
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import check_implementation, \
from meow_base.functionality.validation import check_implementation, \
valid_string

View File

@ -1,10 +1,16 @@
"""
This file contains certain meow specific defintions, most notably the
dictionaries passed around within the runner.
Author(s): David Marchant
"""
from datetime import datetime
from typing import Any, Dict, Type
from meow_base.core.rule import Rule
from meow_base.core.correctness.validation import check_type
from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \
from meow_base.functionality.validation import check_type
from meow_base.core.vars import EVENT_TYPE, EVENT_PATH, \
JOB_EVENT, JOB_TYPE, JOB_ID, JOB_PATTERN, JOB_RECIPE, JOB_RULE, \
JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, WATCHDOG_HASH
@ -12,7 +18,6 @@ from meow_base.core.correctness.vars import EVENT_TYPE, EVENT_PATH, \
EVENT_KEYS = {
EVENT_TYPE: str,
EVENT_PATH: str,
# Should be a Rule but can't import here due to circular dependencies
EVENT_RULE: Rule
}

View File

@ -12,9 +12,9 @@ if "BasePattern" not in modules:
from meow_base.core.base_pattern import BasePattern
if "BaseRecipe" not in modules:
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.vars import VALID_RULE_NAME_CHARS, \
from meow_base.core.vars import VALID_RULE_NAME_CHARS, \
get_drt_imp_msg
from meow_base.core.correctness.validation import valid_string, check_type, \
from meow_base.functionality.validation import valid_string, check_type, \
check_implementation
from meow_base.functionality.naming import generate_rule_id

View File

@ -17,10 +17,10 @@ from typing import Any, Union, Dict, List, Type
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.correctness.vars import DEBUG_WARNING, DEBUG_INFO, \
from meow_base.core.vars import DEBUG_WARNING, DEBUG_INFO, \
EVENT_TYPE, VALID_CHANNELS, META_FILE, DEFAULT_JOB_OUTPUT_DIR, \
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH
from meow_base.core.correctness.validation import check_type, valid_list, \
from meow_base.functionality.validation import check_type, valid_list, \
valid_dir_path
from meow_base.functionality.debug import setup_debugging, print_debug
from meow_base.functionality.file_io import make_dir, read_yaml

View File

@ -6,8 +6,8 @@ Author(s): David Marchant
from typing import Any, Tuple
from meow_base.core.correctness.validation import check_type
from meow_base.core.correctness.vars import DEBUG_INFO, DEBUG_WARNING
from meow_base.functionality.validation import check_type
from meow_base.core.vars import DEBUG_INFO, DEBUG_WARNING
def setup_debugging(print:Any=None, logging:int=0)->Tuple[Any,int]:

View File

@ -11,7 +11,7 @@ from os import makedirs, remove, rmdir, walk
from os.path import exists, isfile, join
from typing import Any, Dict, List
from meow_base.core.correctness.validation import valid_path
from meow_base.functionality.validation import valid_path
def make_dir(path:str, can_exist:bool=True, ensure_clean:bool=False):

View File

@ -9,8 +9,8 @@ from os import listdir
from os.path import isfile
from meow_base.core.correctness.vars import HASH_BUFFER_SIZE, SHA256
from meow_base.core.correctness.validation import check_type, \
from meow_base.core.vars import HASH_BUFFER_SIZE, SHA256
from meow_base.functionality.validation import check_type, \
valid_existing_file_path, valid_existing_dir_path
def _get_file_sha256(file_path:str)->str:

View File

@ -11,9 +11,9 @@ from typing import Any, Dict, Union, List
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.rule import Rule
from meow_base.core.correctness.validation import check_type, valid_dict, \
from meow_base.functionality.validation import check_type, valid_dict, \
valid_list
from meow_base.core.correctness.vars import EVENT_PATH, EVENT_RULE, \
from meow_base.core.vars import EVENT_PATH, EVENT_RULE, \
EVENT_TYPE, EVENT_TYPE_WATCHDOG, JOB_CREATE_TIME, JOB_EVENT, JOB_ID, \
JOB_PATTERN, JOB_RECIPE, JOB_REQUIREMENTS, JOB_RULE, JOB_STATUS, \
JOB_TYPE, STATUS_QUEUED, WATCHDOG_BASE, WATCHDOG_HASH, SWEEP_JUMP, \

View File

@ -7,7 +7,7 @@ Author(s): David Marchant
from typing import List
from random import SystemRandom
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
#TODO Make this guaranteed unique

View File

@ -10,7 +10,7 @@ from os import getenv
from papermill.translators import papermill_translators
from typing import Any, Dict, List
from meow_base.core.correctness.validation import check_script, check_type
from meow_base.functionality.validation import check_script, check_type
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any],
@ -151,4 +151,4 @@ def parameterize_bash_script(script:List[str], parameters:Dict[str,Any],
# Validate that the parameterized notebook is still valid
check_script(output_script)
return output_script
return output_script

View File

@ -12,7 +12,7 @@ from multiprocessing.connection import Connection, wait as multi_wait
if osName == 'nt':
from multiprocessing.connection import PipeConnection
from multiprocessing.queues import Queue
from meow_base.core.correctness.vars import VALID_CHANNELS
from meow_base.core.vars import VALID_CHANNELS
def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:

View File

@ -11,7 +11,7 @@ from os.path import basename
from sys import version_info, prefix, base_prefix
from typing import Any, Dict, List, Tuple, Union
from meow_base.core.correctness.validation import check_type
from meow_base.functionality.validation import check_type
REQUIREMENT_PYTHON = "python"
REQ_PYTHON_MODULES = "modules"

View File

@ -11,7 +11,7 @@ from os.path import sep, exists, isfile, isdir, dirname
from typing import Any, _SpecialForm, Union, Type, Dict, List, \
get_origin, get_args
from meow_base.core.correctness.vars import VALID_PATH_CHARS, get_not_imp_msg
from meow_base.core.vars import VALID_PATH_CHARS, get_not_imp_msg
def check_type(variable:Any, expected_type:Type, alt_types:List[Type]=[],
or_none:bool=False, hint:str="")->None:

View File

@ -22,9 +22,9 @@ from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.rule import Rule
from meow_base.core.correctness.validation import check_type, valid_string, \
from meow_base.functionality.validation import check_type, valid_string, \
valid_dict, valid_list, valid_dir_path
from meow_base.core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
from meow_base.core.vars import VALID_RECIPE_NAME_CHARS, \
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, DEBUG_INFO, DIR_EVENTS, \
FILE_RETROACTIVE_EVENT, SHA256, VALID_PATH_CHARS, FILE_CLOSED_EVENT, \

View File

@ -7,10 +7,10 @@ from typing import Any, Dict, List, Tuple
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_type, valid_dict, \
from meow_base.core.meow import valid_event
from meow_base.functionality.validation import check_type, valid_dict, \
valid_string, valid_dir_path
from meow_base.core.correctness.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
from meow_base.core.vars import DEBUG_INFO, DEFAULT_JOB_QUEUE_DIR, \
VALID_VARIABLE_NAME_CHARS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, JOB_ID, \
EVENT_TYPE_WATCHDOG, JOB_TYPE_BASH, JOB_PARAMETERS, WATCHDOG_HASH, \
WATCHDOG_BASE, META_FILE, STATUS_QUEUED, JOB_STATUS, \

View File

@ -13,10 +13,10 @@ from typing import Any, Tuple, Dict
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_type, valid_string, \
from meow_base.core.meow import valid_event
from meow_base.functionality.validation import check_type, valid_string, \
valid_dict, valid_path, valid_dir_path, valid_existing_file_path
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \
from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
DEFAULT_JOB_QUEUE_DIR, EVENT_PATH, JOB_TYPE_PAPERMILL, WATCHDOG_HASH, \
JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, PARAMS_FILE, \
@ -203,7 +203,7 @@ def papermill_job_func(job_dir):
import os
import papermill
from datetime import datetime
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
from meow_base.core.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_job_file, \

View File

@ -12,10 +12,10 @@ from typing import Any, Tuple, Dict, List
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_handler import BaseHandler
from meow_base.core.correctness.meow import valid_event
from meow_base.core.correctness.validation import check_script, valid_string, \
from meow_base.core.meow import valid_event
from meow_base.functionality.validation import check_script, valid_string, \
valid_dict, valid_dir_path
from meow_base.core.correctness.vars import VALID_VARIABLE_NAME_CHARS, \
from meow_base.core.vars import VALID_VARIABLE_NAME_CHARS, \
PYTHON_FUNC, DEBUG_INFO, EVENT_TYPE_WATCHDOG, \
DEFAULT_JOB_QUEUE_DIR, EVENT_RULE, EVENT_PATH, JOB_TYPE_PYTHON, \
WATCHDOG_HASH, JOB_PARAMETERS, JOB_ID, WATCHDOG_BASE, META_FILE, \
@ -179,7 +179,7 @@ def python_job_func(job_dir):
import os
from datetime import datetime
from io import StringIO
from meow_base.core.correctness.vars import JOB_EVENT, JOB_ID, \
from meow_base.core.vars import JOB_EVENT, JOB_ID, \
EVENT_PATH, META_FILE, PARAMS_FILE, \
JOB_STATUS, SHA256, STATUS_SKIPPED, JOB_END_TIME, \
JOB_ERROR, STATUS_FAILED, get_base_file, \

View File

@ -7,7 +7,7 @@ import os
from distutils.dir_util import copy_tree
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, \
from meow_base.core.vars import DEFAULT_JOB_OUTPUT_DIR, \
DEFAULT_JOB_QUEUE_DIR
from meow_base.functionality.file_io import make_dir, rmtree
from meow_base.patterns.file_event_pattern import FileEventPattern

View File

@ -11,10 +11,8 @@ skip_long_tests=0
for arg in "$@"
do
echo "Given arg: $arg";
if [[ $arg == -s ]]
then
echo "skippy the kangaroo"
skip_long_tests=1
fi
done
@ -31,7 +29,6 @@ do
&& [[ $entry != ./shared.py ]];
then
SKIP_LONG=$skip_long_tests pytest $entry "-W ignore::DeprecationWarning"
# SKIP_LONG=$skip_long_tests pytest $entry
fi
done

View File

@ -8,7 +8,7 @@ from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from meow_base.core.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from meow_base.patterns.file_event_pattern import FileEventPattern
from shared import setup, teardown

View File

@ -6,7 +6,7 @@ import unittest
from datetime import datetime
from typing import Dict
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, \
from meow_base.core.vars import JOB_TYPE_PYTHON, SHA256, \
JOB_PARAMETERS, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, \
JOB_EVENT, META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, \
JOB_PATTERN, STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, \

View File

@ -12,7 +12,7 @@ from time import sleep
from typing import Dict
from meow_base.core.rule import Rule
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
from meow_base.core.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, \

View File

@ -6,7 +6,7 @@ import unittest
from multiprocessing import Pipe
from time import sleep
from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
from meow_base.core.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
SWEEP_JUMP, SWEEP_STOP, DIR_EVENTS
from meow_base.functionality.file_io import make_dir

View File

@ -8,8 +8,8 @@ import unittest
from multiprocessing import Pipe
from typing import Dict
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, \
from meow_base.core.meow import valid_job
from meow_base.core.vars import EVENT_TYPE, WATCHDOG_BASE, \
EVENT_RULE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, \
JOB_ID, JOB_TYPE_PYTHON, JOB_PARAMETERS, PYTHON_FUNC, \
JOB_STATUS, META_FILE, JOB_ERROR, PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, \

View File

@ -13,7 +13,7 @@ from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.conductors import LocalPythonConductor
from meow_base.core.correctness.vars import JOB_TYPE_PAPERMILL, JOB_ERROR, \
from meow_base.core.vars import JOB_TYPE_PAPERMILL, JOB_ERROR, \
META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME, get_result_file
from meow_base.core.runner import MeowRunner
from meow_base.functionality.file_io import make_dir, read_file, \
@ -972,7 +972,7 @@ class MeowTests(unittest.TestCase):
# Test some actual scientific analysis, but in a simple progression
def testScientificAnalysisAllGood(self)->None:
if os.environ["SKIP_LONG"]:
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
warn("Skipping testScientificAnalysisAllGood")
return
@ -1096,7 +1096,7 @@ class MeowTests(unittest.TestCase):
# Test some actual scientific analysis, in a predicatable loop
def testScientificAnalysisPredictableLoop(self)->None:
if os.environ["SKIP_LONG"]:
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
warn("Skipping testScientificAnalysisPredictableLoop")
return
@ -1235,7 +1235,7 @@ class MeowTests(unittest.TestCase):
# Test some actual scientific analysis, in an unpredicatable loop
def testScientificAnalysisRandomLoop(self)->None:
if os.environ["SKIP_LONG"]:
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
warn("Skipping testScientificAnalysisRandomLoop")
return
@ -1387,7 +1387,7 @@ class MeowTests(unittest.TestCase):
# Test some actual scientific analysis, in an unpredicatable loop
def testScientificAnalysisMassiveRandomLoop(self)->None:
if os.environ["SKIP_LONG"]:
if os.environ["SKIP_LONG"] and os.environ["SKIP_LONG"] == '1':
warn("Skipping testScientificAnalysisMassiveRandomLoop")
return

View File

@ -5,13 +5,13 @@ import os
from datetime import datetime
from typing import Any, Union
from meow_base.core.correctness.meow import valid_event, valid_job, \
from meow_base.core.meow import valid_event, valid_job, \
valid_watchdog_event
from meow_base.core.correctness.validation import check_type, \
from meow_base.functionality.validation import check_type, \
check_implementation, valid_string, valid_dict, valid_list, \
valid_existing_file_path, valid_dir_path, valid_non_existing_path, \
check_callable
from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, \
from meow_base.core.vars import VALID_NAME_CHARS, SHA256, \
EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, \
WATCHDOG_BASE, WATCHDOG_HASH