diff --git a/core/correctness/vars.py b/core/correctness/vars.py index 52fb666..7956eff 100644 --- a/core/correctness/vars.py +++ b/core/correctness/vars.py @@ -38,162 +38,6 @@ VALID_CHANNELS = Union[Connection,Queue] HASH_BUFFER_SIZE = 65536 SHA256 = "sha256" -# testing -BAREBONES_NOTEBOOK = { - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 4 -} -TEST_MONITOR_BASE = "test_monitor_base" -TEST_HANDLER_BASE = "test_handler_base" -TEST_JOB_OUTPUT = "test_job_output" -COMPLETE_NOTEBOOK = { - "cells": [ - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": "# The first cell\n\ns = 0\nnum = 1000" - }, - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": "for i in range(num):\n s += i" - }, - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": "div_by = 4" - }, - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": "result = s / div_by" - }, - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": "print(result)" - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} -APPENDING_NOTEBOOK = { - "cells": [ - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": [ - "# Default parameters values\n", - "# The line to append\n", - "extra = 'This line comes from a default pattern'\n", - "# Data input file location\n", - "infile = 'start/alpha.txt'\n", - "# Output file location\n", - "outfile = 'first/alpha.txt'" - ] - }, - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": [ - "# load in dataset. This should be a text file\n", - "with open(infile) as input_file:\n", - " data = input_file.read()" - ] - }, - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": [ - "# Append the line\n", - "appended = data + '\\n' + extra" - ] - }, - { - "cell_type": "code", - "execution_count": None, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Create output directory if it doesn't exist\n", - "output_dir_path = os.path.dirname(outfile)\n", - "\n", - "if output_dir_path:\n", - " os.makedirs(output_dir_path, exist_ok=True)\n", - "\n", - "# Save added array as new dataset\n", - "with open(outfile, 'w') as output_file:\n", - " output_file.write(appended)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6 (main, Nov 14 2022, 16:10:14) [GCC 11.3.0]" - }, - "vscode": { - "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} - # meow events EVENT_TYPE = "event_type" EVENT_PATH = "event_path" diff --git a/tests/shared.py b/tests/shared.py index 735c242..d4d3fca 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -1,11 +1,15 @@ """ -This file contains shared functions used within multiple tests. +This file contains shared functions and variables used within multiple tests. Author(s): David Marchant """ from core.functionality import make_dir, rmtree -from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \ - TEST_MONITOR_BASE + + +# testing +TEST_MONITOR_BASE = "test_monitor_base" +TEST_HANDLER_BASE = "test_handler_base" +TEST_JOB_OUTPUT = "test_job_output" def setup(): make_dir(TEST_MONITOR_BASE, ensure_clean=True) @@ -17,3 +21,156 @@ def teardown(): rmtree(TEST_HANDLER_BASE) rmtree(TEST_JOB_OUTPUT) rmtree("first") + +# Jupyter notebooks +BAREBONES_NOTEBOOK = { + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} +COMPLETE_NOTEBOOK = { + "cells": [ + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": "# The first cell\n\ns = 0\nnum = 1000" + }, + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": "for i in range(num):\n s += i" + }, + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": "div_by = 4" + }, + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": "result = s / div_by" + }, + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": "print(result)" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} +APPENDING_NOTEBOOK = { + "cells": [ + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": [ + "# Default parameters values\n", + "# The line to append\n", + "extra = 'This line comes from a default pattern'\n", + "# Data input file location\n", + "infile = 'start/alpha.txt'\n", + "# Output file location\n", + "outfile = 'first/alpha.txt'" + ] + }, + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": [ + "# load in dataset. This should be a text file\n", + "with open(infile) as input_file:\n", + " data = input_file.read()" + ] + }, + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": [ + "# Append the line\n", + "appended = data + '\\n' + extra" + ] + }, + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Create output directory if it doesn't exist\n", + "output_dir_path = os.path.dirname(outfile)\n", + "\n", + "if output_dir_path:\n", + " os.makedirs(output_dir_path, exist_ok=True)\n", + "\n", + "# Save added array as new dataset\n", + "with open(outfile, 'w') as output_file:\n", + " output_file.write(appended)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6 (main, Nov 14 2022, 16:10:14) [GCC 11.3.0]" + }, + "vscode": { + "interpreter": { + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/test_conductors.py b/tests/test_conductors.py index 03592b1..ddbd498 100644 --- a/tests/test_conductors.py +++ b/tests/test_conductors.py @@ -2,8 +2,7 @@ import os import unittest -from core.correctness.vars import PYTHON_TYPE, TEST_HANDLER_BASE, SHA256, \ - TEST_JOB_OUTPUT, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, WATCHDOG_TYPE, \ +from core.correctness.vars import PYTHON_TYPE, SHA256, WATCHDOG_TYPE, \ WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_HASH, JOB_PARAMETERS, JOB_HASH, \ PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, META_FILE, \ BASE_FILE, PARAMS_FILE, JOB_FILE, RESULT_FILE @@ -12,7 +11,8 @@ from core.meow import create_rule from conductors import LocalPythonConductor from patterns import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, job_func -from shared import setup, teardown +from shared import setup, teardown, TEST_MONITOR_BASE, APPENDING_NOTEBOOK, \ + TEST_JOB_OUTPUT, TEST_HANDLER_BASE def failing_func(): diff --git a/tests/test_functionality.py b/tests/test_functionality.py index cceeebc..959cf54 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -8,11 +8,10 @@ from multiprocessing import Pipe, Queue from time import sleep from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \ - SHA256, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, EVENT_PATH, \ - WATCHDOG_TYPE, PYTHON_TYPE, WATCHDOG_BASE, WATCHDOG_HASH, WATCHDOG_RULE, \ - JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, \ - PYTHON_EXECUTION_BASE, APPENDING_NOTEBOOK, JOB_ID, JOB_EVENT, JOB_TYPE, \ - JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ + SHA256, EVENT_TYPE, EVENT_PATH, WATCHDOG_TYPE, PYTHON_TYPE, \ + WATCHDOG_BASE, WATCHDOG_HASH, WATCHDOG_RULE, JOB_PARAMETERS, JOB_HASH, \ + PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, JOB_ID, JOB_EVENT, \ + JOB_TYPE, JOB_PATTERN, JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME, \ JOB_REQUIREMENTS, STATUS_QUEUED from core.functionality import generate_id, wait, get_file_hash, rmtree, \ make_dir, parameterize_jupyter_notebook, create_event, create_job, \ @@ -23,7 +22,8 @@ from core.functionality import generate_id, wait, get_file_hash, rmtree, \ from core.meow import create_rule from patterns import FileEventPattern from recipes import JupyterNotebookRecipe -from shared import setup, teardown +from shared import setup, teardown, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, \ + APPENDING_NOTEBOOK class CorrectnessTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_meow.py b/tests/test_meow.py index 3ab0144..5ae809b 100644 --- a/tests/test_meow.py +++ b/tests/test_meow.py @@ -3,12 +3,11 @@ import unittest from typing import Any, Union -from core.correctness.vars import BAREBONES_NOTEBOOK from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \ BaseHandler, BaseConductor, create_rules, create_rule from patterns import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe -from shared import setup, teardown +from shared import setup, teardown, BAREBONES_NOTEBOOK valid_pattern_one = FileEventPattern( "pattern_one", "path_one", "recipe_one", "file_one") diff --git a/tests/test_patterns.py b/tests/test_patterns.py index ce31bd5..b4b8826 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -5,14 +5,13 @@ import unittest from multiprocessing import Pipe -from core.correctness.vars import FILE_CREATE_EVENT, BAREBONES_NOTEBOOK, \ - TEST_MONITOR_BASE, EVENT_TYPE, WATCHDOG_RULE, WATCHDOG_BASE, \ - WATCHDOG_TYPE, EVENT_PATH +from core.correctness.vars import FILE_CREATE_EVENT, EVENT_TYPE, \ + WATCHDOG_RULE, WATCHDOG_BASE, WATCHDOG_TYPE, EVENT_PATH from core.functionality import make_dir from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor, \ _DEFAULT_MASK, SWEEP_START, SWEEP_STOP, SWEEP_JUMP from recipes import JupyterNotebookRecipe -from shared import setup, teardown +from shared import setup, teardown, BAREBONES_NOTEBOOK, TEST_MONITOR_BASE def patterns_equal(tester, pattern_one, pattern_two): diff --git a/tests/test_recipes.py b/tests/test_recipes.py index fa3a78c..29dc84b 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -5,12 +5,10 @@ import unittest from multiprocessing import Pipe -from core.correctness.vars import BAREBONES_NOTEBOOK, TEST_HANDLER_BASE, \ - TEST_JOB_OUTPUT, TEST_MONITOR_BASE, COMPLETE_NOTEBOOK, EVENT_TYPE, \ - WATCHDOG_BASE, WATCHDOG_RULE, WATCHDOG_TYPE, EVENT_PATH, SHA256, \ - WATCHDOG_HASH, JOB_ID, PYTHON_TYPE, JOB_PARAMETERS, JOB_HASH, \ - PYTHON_FUNC, PYTHON_OUTPUT_DIR, PYTHON_EXECUTION_BASE, \ - APPENDING_NOTEBOOK, META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, \ +from core.correctness.vars import EVENT_TYPE, WATCHDOG_BASE, WATCHDOG_RULE, \ + WATCHDOG_TYPE, EVENT_PATH, SHA256, WATCHDOG_HASH, JOB_ID, PYTHON_TYPE, \ + JOB_PARAMETERS, JOB_HASH, PYTHON_FUNC, PYTHON_OUTPUT_DIR, \ + PYTHON_EXECUTION_BASE, META_FILE, BASE_FILE, PARAMS_FILE, JOB_FILE, \ RESULT_FILE from core.correctness.validation import valid_job from core.functionality import get_file_hash, create_job, create_event @@ -19,7 +17,8 @@ from patterns.file_event_pattern import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe, \ PapermillHandler, job_func from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule -from shared import setup, teardown +from shared import setup, teardown, TEST_HANDLER_BASE, TEST_MONITOR_BASE, \ + TEST_JOB_OUTPUT, BAREBONES_NOTEBOOK, APPENDING_NOTEBOOK, COMPLETE_NOTEBOOK class CorrectnessTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_rules.py b/tests/test_rules.py index b4e9358..c147597 100644 --- a/tests/test_rules.py +++ b/tests/test_rules.py @@ -1,11 +1,10 @@ import unittest -from core.correctness.vars import BAREBONES_NOTEBOOK from patterns.file_event_pattern import FileEventPattern from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule -from shared import setup, teardown +from shared import setup, teardown, BAREBONES_NOTEBOOK class CorrectnessTests(unittest.TestCase): def setUp(self)->None: diff --git a/tests/test_runner.py b/tests/test_runner.py index 362bf5d..480ecd3 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -6,15 +6,16 @@ import unittest from time import sleep from conductors import LocalPythonConductor -from core.correctness.vars import TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \ - TEST_MONITOR_BASE, APPENDING_NOTEBOOK, RESULT_FILE +from core.correctness.vars import RESULT_FILE from core.functionality import make_dir, read_notebook from core.meow import BaseMonitor, BaseHandler, BaseConductor from core.runner import MeowRunner -from patterns import WatchdogMonitor, FileEventPattern +from patterns.file_event_pattern import WatchdogMonitor, FileEventPattern, \ + SWEEP_JUMP, SWEEP_START, SWEEP_STOP from recipes.jupyter_notebook_recipe import PapermillHandler, \ JupyterNotebookRecipe -from shared import setup, teardown +from shared import setup, teardown, TEST_HANDLER_BASE, TEST_JOB_OUTPUT, \ + TEST_MONITOR_BASE, APPENDING_NOTEBOOK class MeowTests(unittest.TestCase): @@ -361,3 +362,175 @@ class MeowTests(unittest.TestCase): self.assertEqual(data, "Initial Data\nA line from Pattern 1\nA line from Pattern 2") + + # Test single swept meow job execution + def testMeowRunnerExecution(self)->None: + pattern_one = FileEventPattern( + "pattern_one", "start/A.txt", "recipe_one", "infile", + parameters={ + "extra":"A line from a test Pattern", + "outfile":"{VGRID}/output/{FILENAME}" + }) + recipe = JupyterNotebookRecipe( + "recipe_one", APPENDING_NOTEBOOK) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + runner_debug_stream = io.StringIO("") + + runner = MeowRunner( + WatchdogMonitor( + TEST_MONITOR_BASE, + patterns, + recipes, + settletime=1 + ), + PapermillHandler( + TEST_HANDLER_BASE, + TEST_JOB_OUTPUT, + ), + LocalPythonConductor(), + print=runner_debug_stream, + logging=3 + ) + + runner.start() + + start_dir = os.path.join(TEST_MONITOR_BASE, "start") + make_dir(start_dir) + self.assertTrue(start_dir) + with open(os.path.join(start_dir, "A.txt"), "w") as f: + f.write("Initial Data") + + self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt"))) + + loops = 0 + job_id = None + while loops < 15: + sleep(1) + runner_debug_stream.seek(0) + messages = runner_debug_stream.readlines() + + for msg in messages: + self.assertNotIn("ERROR", msg) + + if "INFO: Completed execution for job: '" in msg: + job_id = msg.replace( + "INFO: Completed execution for job: '", "") + job_id = job_id[:-2] + loops = 15 + loops += 1 + + print("JOB ID:") + print(job_id) + + self.assertIsNotNone(job_id) + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1) + self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT)) + + runner.stop() + + job_dir = os.path.join(TEST_JOB_OUTPUT, job_id) + self.assertEqual(len(os.listdir(job_dir)), 5) + + result = read_notebook(os.path.join(job_dir, RESULT_FILE)) + self.assertIsNotNone(result) + + output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt") + self.assertTrue(os.path.exists(output_path)) + + with open(output_path, "r") as f: + data = f.read() + + self.assertEqual(data, "Initial Data\nA line from a test Pattern") + + # Test multiple swept meow job execution + def testMeowRunnerExecution(self)->None: + pattern_one = FileEventPattern( + "pattern_one", "start/A.txt", "recipe_one", "infile", + parameters={ + "extra":"A line from a test Pattern", + "outfile":"{VGRID}/output/{FILENAME}" + }) + recipe = JupyterNotebookRecipe( + "recipe_one", APPENDING_NOTEBOOK) + + patterns = { + pattern_one.name: pattern_one, + } + recipes = { + recipe.name: recipe, + } + + runner_debug_stream = io.StringIO("") + + runner = MeowRunner( + WatchdogMonitor( + TEST_MONITOR_BASE, + patterns, + recipes, + settletime=1 + ), + PapermillHandler( + TEST_HANDLER_BASE, + TEST_JOB_OUTPUT, + ), + LocalPythonConductor(), + print=runner_debug_stream, + logging=3 + ) + + runner.start() + + start_dir = os.path.join(TEST_MONITOR_BASE, "start") + make_dir(start_dir) + self.assertTrue(start_dir) + with open(os.path.join(start_dir, "A.txt"), "w") as f: + f.write("Initial Data") + + self.assertTrue(os.path.exists(os.path.join(start_dir, "A.txt"))) + + loops = 0 + job_id = None + while loops < 15: + sleep(1) + runner_debug_stream.seek(0) + messages = runner_debug_stream.readlines() + + for msg in messages: + self.assertNotIn("ERROR", msg) + + if "INFO: Completed execution for job: '" in msg: + job_id = msg.replace( + "INFO: Completed execution for job: '", "") + job_id = job_id[:-2] + loops = 15 + loops += 1 + + print("JOB ID:") + print(job_id) + + self.assertIsNotNone(job_id) + self.assertEqual(len(os.listdir(TEST_JOB_OUTPUT)), 1) + self.assertIn(job_id, os.listdir(TEST_JOB_OUTPUT)) + + runner.stop() + + job_dir = os.path.join(TEST_JOB_OUTPUT, job_id) + self.assertEqual(len(os.listdir(job_dir)), 5) + + result = read_notebook(os.path.join(job_dir, RESULT_FILE)) + self.assertIsNotNone(result) + + output_path = os.path.join(TEST_MONITOR_BASE, "output", "A.txt") + self.assertTrue(os.path.exists(output_path)) + + with open(output_path, "r") as f: + data = f.read() + + self.assertEqual(data, "Initial Data\nA line from a test Pattern") diff --git a/tests/test_validation.py b/tests/test_validation.py index 96d02f9..c995dbf 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -10,11 +10,11 @@ from core.correctness.validation import check_type, check_implementation, \ valid_string, valid_dict, valid_list, valid_existing_file_path, \ valid_existing_dir_path, valid_non_existing_path, valid_event, valid_job, \ setup_debugging -from core.correctness.vars import VALID_NAME_CHARS, TEST_MONITOR_BASE, \ - SHA256, EVENT_TYPE, EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, \ - JOB_RECIPE, JOB_RULE, JOB_STATUS, JOB_CREATE_TIME +from core.correctness.vars import VALID_NAME_CHARS, SHA256, EVENT_TYPE, \ + EVENT_PATH, JOB_TYPE, JOB_EVENT, JOB_ID, JOB_PATTERN, JOB_RECIPE, \ + JOB_RULE, JOB_STATUS, JOB_CREATE_TIME from core.functionality import make_dir -from shared import setup, teardown +from shared import setup, teardown, TEST_MONITOR_BASE class CorrectnessTests(unittest.TestCase): def setUp(self)->None: