NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

also refactored core.meow into seperate files in hope that it'll help solve circular imports

Browse Source
This commit is contained in:
PatchOfScotland
2023-02-10 16:23:24 +01:00
parent b8885146fa
commit 6fab2f7a10
22 changed files with 701 additions and 629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tests/shared.py
View File

@ -7,7 +7,8 @@ import os
from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR
from functionality.file_io import make_dir, rmtree
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
# testing
TEST_DIR = "test_files"
@ -15,6 +16,7 @@ TEST_MONITOR_BASE = "test_monitor_base"
TEST_JOB_QUEUE = "test_job_queue_dir"
TEST_JOB_OUTPUT = "test_job_output"
def setup():
make_dir(TEST_DIR, ensure_clean=True)
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
@ -302,3 +304,13 @@ ADDING_NOTEBOOK = {
"nbformat": 4,
"nbformat_minor": 4
}
valid_pattern_one = FileEventPattern(
"pattern_one", "path_one", "recipe_one", "file_one")
valid_pattern_two = FileEventPattern(
"pattern_two", "path_two", "recipe_two", "file_two")
valid_recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
valid_recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)

3
tests/test_conductors.py
View File

@ -11,12 +11,11 @@ from core.correctness.vars import JOB_TYPE_PYTHON, SHA256, JOB_PARAMETERS, \
STATUS_DONE, JOB_TYPE_PAPERMILL, JOB_RECIPE, JOB_RULE, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, EVENT_PATH, EVENT_RULE, EVENT_TYPE, \
EVENT_TYPE_WATCHDOG, get_base_file, get_result_file, get_job_file
from core.meow import create_rule
from conductors import LocalPythonConductor
from functionality.file_io import read_file, read_yaml, write_file, \
write_notebook, write_yaml, lines_to_string, make_dir
from functionality.hashing import get_file_hash
from functionality.meow import create_watchdog_event, create_job
from functionality.meow import create_watchdog_event, create_job, create_rule
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
papermill_job_func

64
tests/test_functionality.py
View File

@ -6,20 +6,21 @@ import os
from datetime import datetime
from multiprocessing import Pipe, Queue
from time import sleep
from typing import Dict
from core.base_rule import BaseRule
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
SHA256, EVENT_TYPE, EVENT_PATH, EVENT_TYPE_WATCHDOG, \
WATCHDOG_BASE, WATCHDOG_HASH, EVENT_RULE, JOB_PARAMETERS, JOB_HASH, \
PYTHON_FUNC, JOB_ID, JOB_EVENT, \
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
JOB_REQUIREMENTS, STATUS_QUEUED, JOB_TYPE_PAPERMILL
from core.meow import create_rule
from functionality.file_io import lines_to_string, make_dir, read_file, \
read_file_lines, read_notebook, read_yaml, rmtree, write_file, \
write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_event, create_job, \
create_watchdog_event, replace_keywords, \
from functionality.meow import create_event, create_job, create_rule, \
create_rules, create_watchdog_event, replace_keywords, \
KEYWORD_BASE, KEYWORD_DIR, KEYWORD_EXTENSION, KEYWORD_FILENAME, \
KEYWORD_JOB, KEYWORD_PATH, KEYWORD_PREFIX, KEYWORD_REL_DIR, \
KEYWORD_REL_PATH
@ -29,8 +30,9 @@ from functionality.parameterisation import parameterize_jupyter_notebook, \
from functionality.process_io import wait
from patterns import FileEventPattern
from recipes import JupyterNotebookRecipe
from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \
APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
from shared import setup, teardown, valid_recipe_two, valid_recipe_one, \
valid_pattern_one, valid_pattern_two, TEST_MONITOR_BASE, \
COMPLETE_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_PYTHON_SCRIPT
class DebugTests(unittest.TestCase):
def setUp(self)->None:
@ -553,6 +555,58 @@ class MeowTests(unittest.TestCase):
self.assertEqual(replaced["M"], "A")
self.assertEqual(replaced["N"], 1)
# Test that create_rule creates a rule from pattern and recipe
def testCreateRule(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
self.assertIsInstance(rule, BaseRule)
with self.assertRaises(ValueError):
rule = create_rule(valid_pattern_one, valid_recipe_two)
# Test that create_rules creates nothing from nothing
def testCreateRulesMinimum(self)->None:
rules = create_rules({}, {})
self.assertEqual(len(rules), 0)
# Test that create_rules creates rules from patterns and recipes
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,
valid_pattern_two.name: valid_pattern_two
}
recipes = {
valid_recipe_one.name: valid_recipe_one,
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, Dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
self.assertIsInstance(rule, BaseRule)
self.assertEqual(k, rule.name)
# Test that create_rules creates nothing from invalid pattern inputs
def testCreateRulesMisindexedPatterns(self)->None:
patterns = {
valid_pattern_two.name: valid_pattern_one,
valid_pattern_one.name: valid_pattern_two
}
with self.assertRaises(KeyError):
create_rules(patterns, {})
# Test that create_rules creates nothing from invalid recipe inputs
def testCreateRulesMisindexedRecipes(self)->None:
recipes = {
valid_recipe_two.name: valid_recipe_one,
valid_recipe_one.name: valid_recipe_two
}
with self.assertRaises(KeyError):
create_rules({}, recipes)
class NamingTests(unittest.TestCase):
def setUp(self)->None:

73
tests/test_meow.py
View File

@ -3,24 +3,18 @@ import unittest
from typing import Any, Union, Tuple, Dict
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from core.base_pattern import BasePattern
from core.base_recipe import BaseRecipe
from core.base_rule import BaseRule
from core.correctness.vars import SWEEP_STOP, SWEEP_JUMP, SWEEP_START
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler, BaseConductor, create_rules, create_rule
from patterns import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from shared import setup, teardown, BAREBONES_NOTEBOOK
valid_pattern_one = FileEventPattern(
"pattern_one", "path_one", "recipe_one", "file_one")
valid_pattern_two = FileEventPattern(
"pattern_two", "path_two", "recipe_two", "file_two")
valid_recipe_one = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
valid_recipe_two = JupyterNotebookRecipe(
"recipe_two", BAREBONES_NOTEBOOK)
from shared import setup, teardown, valid_pattern_one, valid_recipe_one
# TODO split me
class MeowTests(unittest.TestCase):
def setUp(self)->None:
super().setUp()
@ -162,57 +156,6 @@ class MeowTests(unittest.TestCase):
pass
FullRule("name", valid_pattern_one, valid_recipe_one)
# Test that create_rule creates a rule from pattern and recipe
def testCreateRule(self)->None:
rule = create_rule(valid_pattern_one, valid_recipe_one)
self.assertIsInstance(rule, BaseRule)
with self.assertRaises(ValueError):
rule = create_rule(valid_pattern_one, valid_recipe_two)
# Test that create_rules creates nothing from nothing
def testCreateRulesMinimum(self)->None:
rules = create_rules({}, {})
self.assertEqual(len(rules), 0)
# Test that create_rules creates rules from patterns and recipes
def testCreateRulesPatternsAndRecipesDicts(self)->None:
patterns = {
valid_pattern_one.name: valid_pattern_one,
valid_pattern_two.name: valid_pattern_two
}
recipes = {
valid_recipe_one.name: valid_recipe_one,
valid_recipe_two.name: valid_recipe_two
}
rules = create_rules(patterns, recipes)
self.assertIsInstance(rules, Dict)
self.assertEqual(len(rules), 2)
for k, rule in rules.items():
self.assertIsInstance(k, str)
self.assertIsInstance(rule, BaseRule)
self.assertEqual(k, rule.name)
# Test that create_rules creates nothing from invalid pattern inputs
def testCreateRulesMisindexedPatterns(self)->None:
patterns = {
valid_pattern_two.name: valid_pattern_one,
valid_pattern_one.name: valid_pattern_two
}
with self.assertRaises(KeyError):
create_rules(patterns, {})
# Test that create_rules creates nothing from invalid recipe inputs
def testCreateRulesMisindexedRecipes(self)->None:
recipes = {
valid_recipe_two.name: valid_recipe_one,
valid_recipe_one.name: valid_recipe_two
}
with self.assertRaises(KeyError):
create_rules({}, recipes)
# Test that BaseMonitor instantiation
def testBaseMonitor(self)->None:
with self.assertRaises(TypeError):

4
tests/test_recipes.py
View File

@ -13,11 +13,11 @@ from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, EVENT_RULE, \
PARAMS_FILE, SWEEP_STOP, SWEEP_JUMP, SWEEP_START, JOB_TYPE_PAPERMILL, \
get_base_file, get_job_file, get_result_file
from core.correctness.validation import valid_job
from core.meow import create_rules, create_rule
from functionality.file_io import lines_to_string, make_dir, read_yaml, \
write_file, write_notebook, write_yaml
from functionality.hashing import get_file_hash
from functionality.meow import create_job, create_watchdog_event
from functionality.meow import create_job, create_rules, create_rule, \
create_watchdog_event
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
PapermillHandler, papermill_job_func

4
tests/test_runner.py
View File

@ -5,10 +5,12 @@ import unittest
from time import sleep
from core.base_conductor import BaseConductor
from core.base_handler import BaseHandler
from core.base_monitor import BaseMonitor
from conductors import LocalPythonConductor
from core.correctness.vars import get_result_file, \
JOB_TYPE_PAPERMILL, JOB_ERROR, META_FILE, JOB_TYPE_PYTHON, JOB_CREATE_TIME
from core.meow import BaseMonitor, BaseHandler, BaseConductor
from core.runner import MeowRunner
from functionality.file_io import make_dir, read_file, read_notebook, read_yaml
from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern