NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reformatted imports to work better on other machines, plus added benchmarking to project

Browse Source
This commit is contained in:
PatchOfScotland
2023-03-13 11:32:45 +01:00
parent c01df1b190
commit 40ed98000b
48 changed files with 907 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/shared.py
View File

@ -7,10 +7,10 @@ import os
from distutils.dir_util import copy_tree
from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from functionality.file_io import make_dir, rmtree
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
from meow_base.core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from meow_base.functionality.file_io import make_dir, rmtree
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
# testing
TEST_DIR = "test_files"

16
tests/test_base.py
View File

@ -3,14 +3,14 @@ import unittest
from typing import Any, Union, Tuple, Dict
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from patterns import FileEventPattern
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.core.base_pattern import BasePattern
from meow_base.core.base_recipe import BaseRecipe
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from meow_base.patterns.file_event_pattern import FileEventPattern
from shared import setup, teardown, valid_pattern_one, valid_recipe_one

16
tests/test_conductors.py
View File

@ -5,21 +5,21 @@ import unittest
from datetime import datetime
from typing import Dict
from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
from meow_base.core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
JOB_HASH, PYTHON_FUNC, JOB_ID, BACKUP_JOB_ERROR_FILE, JOB_EVENT, \
META_FILE, PARAMS_FILE, JOB_STATUS, JOB_ERROR, JOB_TYPE, JOB_PATTERN, \
STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
from conductors import LocalPythonConductor
from functionality.file_io import read_file, read_yaml, write_file, \
from meow_base.conductors import LocalPythonConductor
from meow_base.functionality.file_io import read_file, read_yaml, write_file, \
write_notebook, write_yaml, lines_to_string, make_dir
from functionality.hashing import get_file_hash
from functionality.meow import create_watchdog_event, create_job, create_rule
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_watchdog_event, create_job, create_rule
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func
from recipes.python_recipe import PythonRecipe, python_job_func
from meow_base.recipes.python_recipe import PythonRecipe, python_job_func
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
TEST_JOB_OUTPUT, TEST_JOB_QUEUE, COMPLETE_PYTHON_SCRIPT, \
BAREBONES_PYTHON_SCRIPT, BAREBONES_NOTEBOOK

26
tests/test_functionality.py
View File

@ -11,34 +11,34 @@ from sys import prefix, base_prefix
from time import sleep
from typing import Dict
from core.base_rule import BaseRule
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
from meow_base.core.base_rule import BaseRule
from meow_base.core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from functionality.debug import setup_debugging
from functionality.file_io import lines_to_string, make_dir, read_file, \
from meow_base.functionality.debug import setup_debugging
from meow_base.functionality.file_io import lines_to_string, make_dir, read_file, \
read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_event, create_job, create_rule, \
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_event, create_job, create_rule, \
create_rules, create_watchdog_event, replace_keywords, \
create_parameter_sweep, \
KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
KEYWORD_REL_PATH
from functionality.naming import _generate_id
from functionality.parameterisation import parameterize_jupyter_notebook, \
from meow_base.functionality.naming import _generate_id
from meow_base.functionality.parameterisation import parameterize_jupyter_notebook, \
parameterize_python_script
from functionality.process_io import wait
from functionality.requirements import create_python_requirements, \
from meow_base.functionality.process_io import wait
from meow_base.functionality.requirements import create_python_requirements, \
check_requirements, \
REQUIREMENT_PYTHON, REQ_PYTHON_ENVIRONMENT, REQ_PYTHON_MODULES, \
REQ_PYTHON_VERSION
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
@ -594,7 +594,7 @@ class MeowTests(unittest.TestCase):
self.assertEqual(len(rules), 0)
# Test that create_rules creates rules from patterns and recipes
# Test that create_rules creates rules from meow_base.patterns and recipes
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,

8
tests/test_patterns.py
View File

@ -5,13 +5,13 @@ import unittest
from multiprocessing import Pipe
from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
from meow_base.core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
EVENT_RULE, WATCHDOG_BASE, EVENT_TYPE_WATCHDOG, EVENT_PATH, SWEEP_START, \
SWEEP_JUMP, SWEEP_STOP
from functionality.file_io import make_dir
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
from meow_base.functionality.file_io import make_dir
from meow_base.patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
_DEFAULT_MASK
from recipes import JupyterNotebookRecipe
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE

18
tests/test_recipes.py
View File

@ -6,23 +6,23 @@ import unittest
from multiprocessing import Pipe
from typing import Dict
from core.correctness.meow import valid_job
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
from meow_base.core.correctness.meow import valid_job
from meow_base.core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
EVENT_TYPE_WATCHDOG, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, \
JOB_TYPE_PYTHON, JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, JOB_STATUS, \
META_FILE, JOB_ERROR, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from functionality.file_io import lines_to_string, make_dir, read_yaml, \
from meow_base.functionality.file_io import lines_to_string, make_dir, read_yaml, \
write_file, write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_job, create_rules, create_rule, \
from meow_base.functionality.hashing import get_file_hash
from meow_base.functionality.meow import create_job, create_rules, create_rule, \
create_watchdog_event
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, papermill_job_func
from recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from rules import FileEventJupyterNotebookRule, FileEventPythonRule
from meow_base.recipes.python_recipe import PythonRecipe, PythonHandler, python_job_func
from meow_base.rules import FileEventPythonRule, FileEventJupyterNotebookRule
from shared import setup, teardown, BAREBONES_PYTHON_SCRIPT, \
COMPLETE_PYTHON_SCRIPT, TEST_JOB_QUEUE, TEST_MONITOR_BASE, \
TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK

6
tests/test_rules.py
View File

@ -1,9 +1,9 @@
import unittest
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from meow_base.patterns.file_event_pattern import FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from meow_base.rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from shared import setup, teardown, BAREBONES_NOTEBOOK
class CorrectnessTests(unittest.TestCase):

28
tests/test_runner.py
View File

@ -8,21 +8,21 @@ from random import shuffle
from shutil import copy
from time import sleep
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from conductors import LocalPythonConductor
from core.correctness.vars import get_result_file, \
from meow_base.core.base_conductor import BaseConductor
from meow_base.core.base_handler import BaseHandler
from meow_base.core.base_monitor import BaseMonitor
from meow_base.conductors import LocalPythonConductor
from meow_base.core.correctness.vars import get_result_file, \
JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME
from core.runner import MeowRunner
from functionality.file_io import make_dir, read_file, read_notebook, \
from meow_base.core.runner import MeowRunner
from meow_base.functionality.file_io import make_dir, read_file, read_notebook, \
read_yaml, write_file, lines_to_string
from functionality.meow import create_parameter_sweep
from functionality.requirements import create_python_requirements
from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
from recipes.jupyter_notebook_recipe import PapermillHandler, \
from meow_base.functionality.meow import create_parameter_sweep
from meow_base.functionality.requirements import create_python_requirements
from meow_base.patterns.file_event_pattern import WatchdogMonitor, FileEventPattern
from meow_base.recipes.jupyter_notebook_recipe import PapermillHandler, \
JupyterNotebookRecipe
from recipes.python_recipe import PythonHandler, PythonRecipe
from meow_base.recipes.python_recipe import PythonHandler, PythonRecipe
from shared import setup, teardown, backup_before_teardown, \
TEST_JOB_QUEUE, TEST_JOB_OUTPUT, TEST_MONITOR_BASE, MAKER_RECIPE, \
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT, TEST_DIR, FILTER_RECIPE, \
@ -916,10 +916,6 @@ class MeowTests(unittest.TestCase):
output = read_file(os.path.join(output_path))
self.assertEqual(output, "12505000.0")
def testSelfModifyingAnalysis(self)->None:
maker_pattern = FileEventPattern(
"maker_pattern",

10
tests/test_validation.py
View File

@ -5,16 +5,16 @@ import os
from datetime import datetime
from typing import Any, Union
from core.correctness.meow import valid_event, valid_job, valid_watchdog_event
from core.correctness.validation import check_type, check_implementation, \
from meow_base.core.correctness.meow import valid_event, valid_job, valid_watchdog_event
from meow_base.core.correctness.validation import check_type, check_implementation, \
valid_string, valid_dict, valid_list, valid_existing_file_path, \
valid_dir_path, valid_non_existing_path, check_callable
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
from meow_base.core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, EVENT_RULE, WATCHDOG_BASE, \
WATCHDOG_HASH
from functionality.file_io import make_dir
from functionality.meow import create_rule
from meow_base.functionality.file_io import make_dir
from meow_base.functionality.meow import create_rule
from shared import setup, teardown, TEST_MONITOR_BASE, valid_pattern_one, \
valid_recipe_one