moved notebook declarations to tests/shared.py
This commit is contained in:
@ -38,162 +38,6 @@ VALID_CHANNELS = Union[Connection,Queue]
|
|||||||
HASH_BUFFER_SIZE = 65536
|
HASH_BUFFER_SIZE = 65536
|
||||||
SHA256 = "sha256"
|
SHA256 = "sha256"
|
||||||
|
|
||||||
# testing
|
|
||||||
BAREBONES_NOTEBOOK = {
|
|
||||||
"cells": [],
|
|
||||||
"metadata": {},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 4
|
|
||||||
}
|
|
||||||
TEST_MONITOR_BASE = "test_monitor_base"
|
|
||||||
TEST_HANDLER_BASE = "test_handler_base"
|
|
||||||
TEST_JOB_OUTPUT = "test_job_output"
|
|
||||||
COMPLETE_NOTEBOOK = {
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": "# The first cell\n\ns = 0\nnum = 1000"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": "for i in range(num):\n s += i"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": "div_by = 4"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": "result = s / div_by"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": "print(result)"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.7.3"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 4
|
|
||||||
}
|
|
||||||
APPENDING_NOTEBOOK = {
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Default parameters values\n",
|
|
||||||
"# The line to append\n",
|
|
||||||
"extra = 'This line comes from a default pattern'\n",
|
|
||||||
"# Data input file location\n",
|
|
||||||
"infile = 'start/alpha.txt'\n",
|
|
||||||
"# Output file location\n",
|
|
||||||
"outfile = 'first/alpha.txt'"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# load in dataset. This should be a text file\n",
|
|
||||||
"with open(infile) as input_file:\n",
|
|
||||||
" data = input_file.read()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# Append the line\n",
|
|
||||||
"appended = data + '\\n' + extra"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": None,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"import os\n",
|
|
||||||
"\n",
|
|
||||||
"# Create output directory if it doesn't exist\n",
|
|
||||||
"output_dir_path = os.path.dirname(outfile)\n",
|
|
||||||
"\n",
|
|
||||||
"if output_dir_path:\n",
|
|
||||||
" os.makedirs(output_dir_path, exist_ok=True)\n",
|
|
||||||
"\n",
|
|
||||||
"# Save added array as new dataset\n",
|
|
||||||
"with open(outfile, 'w') as output_file:\n",
|
|
||||||
" output_file.write(appended)"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 3
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python",
|
|
||||||
"pygments_lexer": "ipython3",
|
|
||||||
"version": "3.10.6 (main, Nov 14 2022, 16:10:14) [GCC 11.3.0]"
|
|
||||||
},
|
|
||||||
"vscode": {
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 4
|
|
||||||
}
|
|
||||||
|
|
||||||
# meow events
|
# meow events
|
||||||
EVENT_TYPE = "event_type"
|
EVENT_TYPE = "event_type"
|
||||||
EVENT_PATH = "event_path"
|
EVENT_PATH = "event_path"
|
||||||
|
163
tests/shared.py
163
tests/shared.py
@ -1,11 +1,15 @@
|
|||||||
"""
|
"""
|
||||||
This file contains shared functions used within multiple tests.
|
This file contains shared functions and variables used within multiple tests.
|
||||||
|
|
||||||
Author(s): David Marchant
|
Author(s): David Marchant
|
||||||
"""
|
"""
|
||||||
from core.functionality import make_dir, rmtree
|
from core.functionality import make_dir, rmtree
|
||||||
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
|
|
||||||
TEST_MONITOR_BASE
|
|
||||||
|
# testing
|
||||||
|
TEST_MONITOR_BASE = "test_monitor_base"
|
||||||
|
TEST_HANDLER_BASE = "test_handler_base"
|
||||||
|
TEST_JOB_OUTPUT = "test_job_output"
|
||||||
|
|
||||||
def setup():
|
def setup():
|
||||||
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
|
make_dir(TEST_MONITOR_BASE, ensure_clean=True)
|
||||||
@ -17,3 +21,156 @@ def teardown():
|
|||||||
rmtree(TEST_HANDLER_BASE)
|
rmtree(TEST_HANDLER_BASE)
|
||||||
rmtree(TEST_JOB_OUTPUT)
|
rmtree(TEST_JOB_OUTPUT)
|
||||||
rmtree("first")
|
rmtree("first")
|
||||||
|
|
||||||
|
# Jupyter notebooks
|
||||||
|
BAREBONES_NOTEBOOK = {
|
||||||
|
"cells": [],
|
||||||
|
"metadata": {},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
|
COMPLETE_NOTEBOOK = {
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": "# The first cell\n\ns = 0\nnum = 1000"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": "for i in range(num):\n s += i"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": "div_by = 4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": "result = s / div_by"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": "print(result)"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
|
APPENDING_NOTEBOOK = {
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Default parameters values\n",
|
||||||
|
"# The line to append\n",
|
||||||
|
"extra = 'This line comes from a default pattern'\n",
|
||||||
|
"# Data input file location\n",
|
||||||
|
"infile = 'start/alpha.txt'\n",
|
||||||
|
"# Output file location\n",
|
||||||
|
"outfile = 'first/alpha.txt'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# load in dataset. This should be a text file\n",
|
||||||
|
"with open(infile) as input_file:\n",
|
||||||
|
" data = input_file.read()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Append the line\n",
|
||||||
|
"appended = data + '\\n' + extra"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": None,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"\n",
|
||||||
|
"# Create output directory if it doesn't exist\n",
|
||||||
|
"output_dir_path = os.path.dirname(outfile)\n",
|
||||||
|
"\n",
|
||||||
|
"if output_dir_path:\n",
|
||||||
|
" os.makedirs(output_dir_path, exist_ok=True)\n",
|
||||||
|
"\n",
|
||||||
|
"# Save added array as new dataset\n",
|
||||||
|
"with open(outfile, 'w') as output_file:\n",
|
||||||
|
" output_file.write(appended)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.6 (main, Nov 14 2022, 16:10:14) [GCC 11.3.0]"
|
||||||
|
},
|
||||||
|
"vscode": {
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
||||||
|
@ -2,8 +2,7 @@
|
|||||||
import os
|
import os
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from core.correctness.vars import PYTHON_TYPE, TEST_HANDLER_BASE, SHA256, \
|
from core.correctness.vars import PYTHON_TYPE, SHA256, WATCHDOG_TYPE, \
|
||||||
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, WATCHDOG_TYPE, \
|
|
||||||
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \
|
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \
|
||||||
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
|
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \
|
||||||
BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
|
BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE
|
||||||
@ -12,7 +11,8 @@ from core.meow import create_rule
|
|||||||
from conductors import LocalPythonConductor
|
from conductors import LocalPythonConductor
|
||||||
from patterns import FileEventPattern
|
from patterns import FileEventPattern
|
||||||
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func
|
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \
|
||||||
|
TEST_JOB_OUTPUT, TEST_HANDLER_BASE
|
||||||
|
|
||||||
|
|
||||||
def failing_func():
|
def failing_func():
|
||||||
|
@ -8,11 +8,10 @@ from multiprocessing import Pipe, Queue
|
|||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
|
||||||
SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, EVENT_PATH, \
|
SHA256, EVENT_TYPE, EVENT_PATH, WATCHDOG_TYPE, PYTHON_TYPE, \
|
||||||
WATCHDOG_TYPE, PYTHON_TYPE, WATCHDOG_BASE, WATCHDOG_HASH, WATCHDOG_RULE, \
|
WATCHDOG_BASE, WATCHDOG_HASH, WATCHDOG_RULE, JOB_PARAMETERS, JOB_HASH, \
|
||||||
JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, \
|
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, JOB_EVENT, \
|
||||||
PYTHON_EXECUTION_BASE, APPENDING_NOTEBOOK, JOB_ID, JOB_EVENT, JOB_TYPE, \
|
JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
|
||||||
JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \
|
|
||||||
JOB_REQUIREMENTS, STATUS_QUEUED
|
JOB_REQUIREMENTS, STATUS_QUEUED
|
||||||
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
|
from core.functionality import generate_id, wait, get_file_hash, rmtree, \
|
||||||
make_dir, parameterize_jupyter_notebook, create_event, create_job, \
|
make_dir, parameterize_jupyter_notebook, create_event, create_job, \
|
||||||
@ -23,7 +22,8 @@ from core.functionality import generate_id, wait, get_file_hash, rmtree, \
|
|||||||
from core.meow import create_rule
|
from core.meow import create_rule
|
||||||
from patterns import FileEventPattern
|
from patterns import FileEventPattern
|
||||||
from recipes import JupyterNotebookRecipe
|
from recipes import JupyterNotebookRecipe
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \
|
||||||
|
APPENDING_NOTEBOOK
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self)->None:
|
||||||
|
@ -3,12 +3,11 @@ import unittest
|
|||||||
|
|
||||||
from typing import Any, Union
|
from typing import Any, Union
|
||||||
|
|
||||||
from core.correctness.vars import BAREBONES_NOTEBOOK
|
|
||||||
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
|
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
|
||||||
BaseHandler, BaseConductor, create_rules, create_rule
|
BaseHandler, BaseConductor, create_rules, create_rule
|
||||||
from patterns import FileEventPattern
|
from patterns import FileEventPattern
|
||||||
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
|
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, BAREBONES_NOTEBOOK
|
||||||
|
|
||||||
valid_pattern_one = FileEventPattern(
|
valid_pattern_one = FileEventPattern(
|
||||||
"pattern_one", "path_one", "recipe_one", "file_one")
|
"pattern_one", "path_one", "recipe_one", "file_one")
|
||||||
|
@ -5,14 +5,13 @@ import unittest
|
|||||||
|
|
||||||
from multiprocessing import Pipe
|
from multiprocessing import Pipe
|
||||||
|
|
||||||
from core.correctness.vars import FILE_CREATE_EVENT, BAREBONES_NOTEBOOK, \
|
from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \
|
||||||
TEST_MONITOR_BASE, EVENT_TYPE, WATCHDOG_RULE, WATCHDOG_BASE, \
|
WATCHDOG_RULE, WATCHDOG_BASE, WATCHDOG_TYPE, EVENT_PATH
|
||||||
WATCHDOG_TYPE, EVENT_PATH
|
|
||||||
from core.functionality import make_dir
|
from core.functionality import make_dir
|
||||||
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
|
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \
|
||||||
_DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP
|
_DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP
|
||||||
from recipes import JupyterNotebookRecipe
|
from recipes import JupyterNotebookRecipe
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE
|
||||||
|
|
||||||
|
|
||||||
def patterns_equal(tester, pattern_one, pattern_two):
|
def patterns_equal(tester, pattern_one, pattern_two):
|
||||||
|
@ -5,12 +5,10 @@ import unittest
|
|||||||
|
|
||||||
from multiprocessing import Pipe
|
from multiprocessing import Pipe
|
||||||
|
|
||||||
from core.correctness.vars import BAREBONES_NOTEBOOK, TEST_HANDLER_BASE, \
|
from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, WATCHDOG_RULE, \
|
||||||
TEST_JOB_OUTPUT, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, \
|
WATCHDOG_TYPE, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, PYTHON_TYPE, \
|
||||||
WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_TYPE, EVENT_PATH, SHA256, \
|
JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, \
|
||||||
WATCHDOG_HASH, JOB_ID, PYTHON_TYPE, JOB_PARAMETERS, JOB_HASH, \
|
PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, \
|
||||||
PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, \
|
|
||||||
APPENDING_NOTEBOOK, META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, \
|
|
||||||
RESULT_FILE
|
RESULT_FILE
|
||||||
from core.correctness.validation import valid_job
|
from core.correctness.validation import valid_job
|
||||||
from core.functionality import get_file_hash, create_job, create_event
|
from core.functionality import get_file_hash, create_job, create_event
|
||||||
@ -19,7 +17,8 @@ from patterns.file_event_pattern import FileEventPattern
|
|||||||
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
|
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \
|
||||||
PapermillHandler, job_func
|
PapermillHandler, job_func
|
||||||
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
|
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \
|
||||||
|
TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self)->None:
|
||||||
|
@ -1,11 +1,10 @@
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from core.correctness.vars import BAREBONES_NOTEBOOK
|
|
||||||
from patterns.file_event_pattern import FileEventPattern
|
from patterns.file_event_pattern import FileEventPattern
|
||||||
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
|
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
|
||||||
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
|
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, BAREBONES_NOTEBOOK
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self)->None:
|
||||||
|
@ -6,15 +6,16 @@ import unittest
|
|||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
from conductors import LocalPythonConductor
|
from conductors import LocalPythonConductor
|
||||||
from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
|
from core.correctness.vars import RESULT_FILE
|
||||||
TEST_MONITOR_BASE, APPENDING_NOTEBOOK, RESULT_FILE
|
|
||||||
from core.functionality import make_dir, read_notebook
|
from core.functionality import make_dir, read_notebook
|
||||||
from core.meow import BaseMonitor, BaseHandler, BaseConductor
|
from core.meow import BaseMonitor, BaseHandler, BaseConductor
|
||||||
from core.runner import MeowRunner
|
from core.runner import MeowRunner
|
||||||
from patterns import WatchdogMonitor, FileEventPattern
|
from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern, \
|
||||||
|
SWEEP_JUMP, SWEEP_START, SWEEP_STOP
|
||||||
from recipes.jupyter_notebook_recipe import PapermillHandler, \
|
from recipes.jupyter_notebook_recipe import PapermillHandler, \
|
||||||
JupyterNotebookRecipe
|
JupyterNotebookRecipe
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \
|
||||||
|
TEST_MONITOR_BASE, APPENDING_NOTEBOOK
|
||||||
|
|
||||||
|
|
||||||
class MeowTests(unittest.TestCase):
|
class MeowTests(unittest.TestCase):
|
||||||
@ -361,3 +362,175 @@ class MeowTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertEqual(data,
|
self.assertEqual(data,
|
||||||
"Initial Data\nA line from Pattern 1\nA line from Pattern 2")
|
"Initial Data\nA line from Pattern 1\nA line from Pattern 2")
|
||||||
|
|
||||||
|
# Test single swept meow job execution
|
||||||
|
def testMeowRunnerExecution(self)->None:
|
||||||
|
pattern_one = FileEventPattern(
|
||||||
|
"pattern_one", "start/A.txt", "recipe_one", "infile",
|
||||||
|
parameters={
|
||||||
|
"extra":"A line from a test Pattern",
|
||||||
|
"outfile":"{VGRID}/output/{FILENAME}"
|
||||||
|
})
|
||||||
|
recipe = JupyterNotebookRecipe(
|
||||||
|
"recipe_one", APPENDING_NOTEBOOK)
|
||||||
|
|
||||||
|
patterns = {
|
||||||
|
pattern_one.name: pattern_one,
|
||||||
|
}
|
||||||
|
recipes = {
|
||||||
|
recipe.name: recipe,
|
||||||
|
}
|
||||||
|
|
||||||
|
runner_debug_stream = io.StringIO("")
|
||||||
|
|
||||||
|
runner = MeowRunner(
|
||||||
|
WatchdogMonitor(
|
||||||
|
TEST_MONITOR_BASE,
|
||||||
|
patterns,
|
||||||
|
recipes,
|
||||||
|
settletime=1
|
||||||
|
),
|
||||||
|
PapermillHandler(
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT,
|
||||||
|
),
|
||||||
|
LocalPythonConductor(),
|
||||||
|
print=runner_debug_stream,
|
||||||
|
logging=3
|
||||||
|
)
|
||||||
|
|
||||||
|
runner.start()
|
||||||
|
|
||||||
|
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
|
||||||
|
make_dir(start_dir)
|
||||||
|
self.assertTrue(start_dir)
|
||||||
|
with open(os.path.join(start_dir, "A.txt"), "w") as f:
|
||||||
|
f.write("Initial Data")
|
||||||
|
|
||||||
|
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
|
||||||
|
|
||||||
|
loops = 0
|
||||||
|
job_id = None
|
||||||
|
while loops < 15:
|
||||||
|
sleep(1)
|
||||||
|
runner_debug_stream.seek(0)
|
||||||
|
messages = runner_debug_stream.readlines()
|
||||||
|
|
||||||
|
for msg in messages:
|
||||||
|
self.assertNotIn("ERROR", msg)
|
||||||
|
|
||||||
|
if "INFO: Completed execution for job: '" in msg:
|
||||||
|
job_id = msg.replace(
|
||||||
|
"INFO: Completed execution for job: '", "")
|
||||||
|
job_id = job_id[:-2]
|
||||||
|
loops = 15
|
||||||
|
loops += 1
|
||||||
|
|
||||||
|
print("JOB ID:")
|
||||||
|
print(job_id)
|
||||||
|
|
||||||
|
self.assertIsNotNone(job_id)
|
||||||
|
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
|
||||||
|
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
|
||||||
|
|
||||||
|
runner.stop()
|
||||||
|
|
||||||
|
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
|
||||||
|
self.assertEqual(len(os.listdir(job_dir)), 5)
|
||||||
|
|
||||||
|
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
|
||||||
|
output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
|
||||||
|
self.assertTrue(os.path.exists(output_path))
|
||||||
|
|
||||||
|
with open(output_path, "r") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
self.assertEqual(data, "Initial Data\nA line from a test Pattern")
|
||||||
|
|
||||||
|
# Test multiple swept meow job execution
|
||||||
|
def testMeowRunnerExecution(self)->None:
|
||||||
|
pattern_one = FileEventPattern(
|
||||||
|
"pattern_one", "start/A.txt", "recipe_one", "infile",
|
||||||
|
parameters={
|
||||||
|
"extra":"A line from a test Pattern",
|
||||||
|
"outfile":"{VGRID}/output/{FILENAME}"
|
||||||
|
})
|
||||||
|
recipe = JupyterNotebookRecipe(
|
||||||
|
"recipe_one", APPENDING_NOTEBOOK)
|
||||||
|
|
||||||
|
patterns = {
|
||||||
|
pattern_one.name: pattern_one,
|
||||||
|
}
|
||||||
|
recipes = {
|
||||||
|
recipe.name: recipe,
|
||||||
|
}
|
||||||
|
|
||||||
|
runner_debug_stream = io.StringIO("")
|
||||||
|
|
||||||
|
runner = MeowRunner(
|
||||||
|
WatchdogMonitor(
|
||||||
|
TEST_MONITOR_BASE,
|
||||||
|
patterns,
|
||||||
|
recipes,
|
||||||
|
settletime=1
|
||||||
|
),
|
||||||
|
PapermillHandler(
|
||||||
|
TEST_HANDLER_BASE,
|
||||||
|
TEST_JOB_OUTPUT,
|
||||||
|
),
|
||||||
|
LocalPythonConductor(),
|
||||||
|
print=runner_debug_stream,
|
||||||
|
logging=3
|
||||||
|
)
|
||||||
|
|
||||||
|
runner.start()
|
||||||
|
|
||||||
|
start_dir = os.path.join(TEST_MONITOR_BASE, "start")
|
||||||
|
make_dir(start_dir)
|
||||||
|
self.assertTrue(start_dir)
|
||||||
|
with open(os.path.join(start_dir, "A.txt"), "w") as f:
|
||||||
|
f.write("Initial Data")
|
||||||
|
|
||||||
|
self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt")))
|
||||||
|
|
||||||
|
loops = 0
|
||||||
|
job_id = None
|
||||||
|
while loops < 15:
|
||||||
|
sleep(1)
|
||||||
|
runner_debug_stream.seek(0)
|
||||||
|
messages = runner_debug_stream.readlines()
|
||||||
|
|
||||||
|
for msg in messages:
|
||||||
|
self.assertNotIn("ERROR", msg)
|
||||||
|
|
||||||
|
if "INFO: Completed execution for job: '" in msg:
|
||||||
|
job_id = msg.replace(
|
||||||
|
"INFO: Completed execution for job: '", "")
|
||||||
|
job_id = job_id[:-2]
|
||||||
|
loops = 15
|
||||||
|
loops += 1
|
||||||
|
|
||||||
|
print("JOB ID:")
|
||||||
|
print(job_id)
|
||||||
|
|
||||||
|
self.assertIsNotNone(job_id)
|
||||||
|
self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1)
|
||||||
|
self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT))
|
||||||
|
|
||||||
|
runner.stop()
|
||||||
|
|
||||||
|
job_dir = os.path.join(TEST_JOB_OUTPUT, job_id)
|
||||||
|
self.assertEqual(len(os.listdir(job_dir)), 5)
|
||||||
|
|
||||||
|
result = read_notebook(os.path.join(job_dir, RESULT_FILE))
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
|
||||||
|
output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt")
|
||||||
|
self.assertTrue(os.path.exists(output_path))
|
||||||
|
|
||||||
|
with open(output_path, "r") as f:
|
||||||
|
data = f.read()
|
||||||
|
|
||||||
|
self.assertEqual(data, "Initial Data\nA line from a test Pattern")
|
||||||
|
@ -10,11 +10,11 @@ from core.correctness.validation import check_type, check_implementation, \
|
|||||||
valid_string, valid_dict, valid_list, valid_existing_file_path, \
|
valid_string, valid_dict, valid_list, valid_existing_file_path, \
|
||||||
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
|
valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \
|
||||||
setup_debugging
|
setup_debugging
|
||||||
from core.correctness.vars import VALID_NAME_CHARS, TEST_MONITOR_BASE, \
|
from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \
|
||||||
SHA256, EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \
|
EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \
|
||||||
JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME
|
JOB_RULE, JOB_STATUS, JOB_CREATE_TIME
|
||||||
from core.functionality import make_dir
|
from core.functionality import make_dir
|
||||||
from shared import setup, teardown
|
from shared import setup, teardown, TEST_MONITOR_BASE
|
||||||
|
|
||||||
class CorrectnessTests(unittest.TestCase):
|
class CorrectnessTests(unittest.TestCase):
|
||||||
def setUp(self)->None:
|
def setUp(self)->None:
|
||||||
|
Reference in New Issue
Block a user