""" This file contains shared functions and variables used within multiple tests. Author(s): David Marchant """ import os from core.correctness.vars import DEFAULT_JOB_OUTPUT_DIR, DEFAULT_JOB_QUEUE_DIR from functionality.file_io import make_dir, rmtree from patterns import FileEventPattern from recipes import JupyterNotebookRecipe # testing TEST_DIR = "test_files" TEST_MONITOR_BASE = "test_monitor_base" TEST_JOB_QUEUE = "test_job_queue_dir" TEST_JOB_OUTPUT = "test_job_output" def setup(): make_dir(TEST_DIR, ensure_clean=True) make_dir(TEST_MONITOR_BASE, ensure_clean=True) make_dir(TEST_JOB_QUEUE, ensure_clean=True) make_dir(TEST_JOB_OUTPUT, ensure_clean=True) make_dir(DEFAULT_JOB_OUTPUT_DIR, ensure_clean=True) make_dir(DEFAULT_JOB_QUEUE_DIR, ensure_clean=True) def teardown(): rmtree(TEST_DIR) rmtree(TEST_MONITOR_BASE) rmtree(TEST_JOB_QUEUE) rmtree(TEST_JOB_OUTPUT) rmtree(DEFAULT_JOB_OUTPUT_DIR) rmtree(DEFAULT_JOB_QUEUE_DIR) rmtree("first") # Recipe funcs BAREBONES_PYTHON_SCRIPT = [ "" ] COMPLETE_PYTHON_SCRIPT = [ "import os", "# Setup parameters", "num = 1000", "infile = 'somehere"+ os.path.sep +"particular'", "outfile = 'nowhere"+ os.path.sep +"particular'", "", "with open(infile, 'r') as file:", " s = float(file.read())", "" "for i in range(num):", " s += i", "", "div_by = 4", "result = s / div_by", "", "print(result)", "", "os.makedirs(os.path.dirname(outfile), exist_ok=True)", "", "with open(outfile, 'w') as file:", " file.write(str(result))", "", "print('done')" ] # Jupyter notebooks BAREBONES_NOTEBOOK = { "cells": [], "metadata": {}, "nbformat": 4, "nbformat_minor": 4 } COMPLETE_NOTEBOOK = { "cells": [ { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "# The first cell\n\ns = 0\nnum = 1000" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "for i in range(num):\n s += i" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "div_by = 4" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "result = s / div_by" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "print(result)" } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 } APPENDING_NOTEBOOK = { "cells": [ { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# Default parameters values\n", "# The line to append\n", "extra = 'This line comes from a default pattern'\n", "# Data input file location\n", "infile = 'start"+ os.path.sep +"alpha.txt'\n", "# Output file location\n", "outfile = 'first"+ os.path.sep +"alpha.txt'" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# load in dataset. This should be a text file\n", "with open(infile) as input_file:\n", " data = input_file.read()" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# Append the line\n", "appended = data + '\\n' + extra" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "# Create output directory if it doesn't exist\n", "output_dir_path = os.path.dirname(outfile)\n", "\n", "if output_dir_path:\n", " os.makedirs(output_dir_path, exist_ok=True)\n", "\n", "# Save added array as new dataset\n", "with open(outfile, 'w') as output_file:\n", " output_file.write(appended)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6 (main, Nov 14 2022, 16:10:14) [GCC 11.3.0]" }, "vscode": { "interpreter": { "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" } } }, "nbformat": 4, "nbformat_minor": 4 } ADDING_NOTEBOOK = { "cells": [ { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# Default parameters values\n", "# Amount to add to data\n", "extra = 10\n", "# Data input file location\n", "infile = 'example_data"+ os.path.sep +"data_0.npy'\n", "# Output file location\n", "outfile = 'standard_output"+ os.path.sep +"data_0.npy'" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import os" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# load in dataset. Should be numpy array\n", "data = np.load(infile)\n" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# Add an amount to all the values in the array\n", "added = data + int(float(extra))\n", "\n", "added" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# Create output directory if it doesn't exist\n", "output_dir_path = os.path.dirname(outfile)\n", "\n", "if output_dir_path:\n", " os.makedirs(output_dir_path, exist_ok=True)\n", "\n", "# Save added array as new dataset\n", "np.save(outfile, added)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" } }, "nbformat": 4, "nbformat_minor": 4 } valid_pattern_one = FileEventPattern( "pattern_one", "path_one", "recipe_one", "file_one") valid_pattern_two = FileEventPattern( "pattern_two", "path_two", "recipe_two", "file_two") valid_recipe_one = JupyterNotebookRecipe( "recipe_one", BAREBONES_NOTEBOOK) valid_recipe_two = JupyterNotebookRecipe( "recipe_two", BAREBONES_NOTEBOOK)