import os from multiprocessing import Queue from multiprocessing.connection import Connection from inspect import signature from typing import Union # validation CHAR_LOWERCASE = 'abcdefghijklmnopqrstuvwxyz' CHAR_UPPERCASE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' CHAR_NUMERIC = '0123456789' VALID_NAME_CHARS = CHAR_UPPERCASE + CHAR_LOWERCASE + CHAR_NUMERIC + "_-" VALID_RECIPE_NAME_CHARS = VALID_NAME_CHARS VALID_PATTERN_NAME_CHARS = VALID_NAME_CHARS VALID_RULE_NAME_CHARS = VALID_NAME_CHARS VALID_VARIABLE_NAME_CHARS = CHAR_UPPERCASE + CHAR_LOWERCASE + CHAR_NUMERIC + "_" VALID_JUPYTER_NOTEBOOK_FILENAME_CHARS = VALID_NAME_CHARS + "." + os.path.sep VALID_JUPYTER_NOTEBOOK_EXTENSIONS = [".ipynb"] VALID_PATH_CHARS = VALID_NAME_CHARS + "." + os.path.sep VALID_TRIGGERING_PATH_CHARS = VALID_NAME_CHARS + ".*" + os.path.sep VALID_CHANNELS = Union[Connection,Queue] # hashing HASH_BUFFER_SIZE = 65536 SHA256 = "sha256" # testing BAREBONES_NOTEBOOK = { "cells": [], "metadata": {}, "nbformat": 4, "nbformat_minor": 4 } TEST_MONITOR_BASE = "test_monitor_base" TEST_HANDLER_BASE = "test_handler_base" TEST_JOB_OUTPUT = "test_job_output" COMPLETE_NOTEBOOK = { "cells": [ { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "# The first cell\n\ns = 0\nnum = 1000" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "for i in range(num):\n s += i" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "div_by = 4" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "result = s / div_by" }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": "print(result)" } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 } APPENDING_NOTEBOOK = { "cells": [ { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# Default parameters values\n", "# The line to append\n", "extra = 'This line comes from a default pattern'\n", "# Data input file location\n", "infile = 'start/alpha.txt'\n", "# Output file location\n", "outfile = 'first/alpha.txt'" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# load in dataset. This should be a text file\n", "with open(infile) as input_file:\n", " data = input_file.read()" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "# Append the line\n", "appended = data + '\\n' + extra" ] }, { "cell_type": "code", "execution_count": None, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "# Create output directory if it doesn't exist\n", "output_dir_path = os.path.dirname(outfile)\n", "\n", "if output_dir_path:\n", " os.makedirs(output_dir_path, exist_ok=True)\n", "\n", "# Save added array as new dataset\n", "with open(outfile, 'w') as output_file:\n", " output_file.write(appended)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6 (main, Nov 14 2022, 16:10:14) [GCC 11.3.0]" }, "vscode": { "interpreter": { "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" } } }, "nbformat": 4, "nbformat_minor": 4 } # meow events EVENT_TYPE = "event_type" EVENT_PATH = "event_path" WATCHDOG_TYPE = "watchdog" WATCHDOG_BASE = "monitor_base" WATCHDOG_RULE = "rule_name" WATCHDOG_HASH = "file_hash" # inotify events FILE_CREATE_EVENT = "file_created" FILE_MODIFY_EVENT = "file_modified" FILE_MOVED_EVENT = "file_moved" FILE_CLOSED_EVENT = "file_closed" FILE_DELETED_EVENT = "file_deleted" FILE_RETROACTIVE_EVENT = "retroactive_file_event" FILE_EVENTS = [ FILE_CREATE_EVENT, FILE_MODIFY_EVENT, FILE_MOVED_EVENT, FILE_CLOSED_EVENT, FILE_DELETED_EVENT, FILE_RETROACTIVE_EVENT ] DIR_CREATE_EVENT = "dir_created" DIR_MODIFY_EVENT = "dir_modified" DIR_MOVED_EVENT = "dir_moved" DIR_DELETED_EVENT = "dir_deleted" DIR_RETROACTIVE_EVENT = "retroactive_dir_event" DIR_EVENTS = [ DIR_CREATE_EVENT, DIR_MODIFY_EVENT, DIR_MOVED_EVENT, DIR_DELETED_EVENT, DIR_RETROACTIVE_EVENT ] # meow jobs JOB_TYPE = "job_type" PYTHON_TYPE = "python" PYTHON_FUNC = "func" PYTHON_EXECUTION_BASE = "exection_base" PYTHON_OUTPUT_DIR = "output_dir" # job definitions JOB_ID = "id" JOB_EVENT = "event" JOB_PATTERN = "pattern" JOB_RECIPE = "recipe" JOB_RULE = "rule" JOB_HASH = "hash" JOB_STATUS = "status" JOB_CREATE_TIME = "create" JOB_START_TIME = "start" JOB_END_TIME = "end" JOB_ERROR = "error" JOB_REQUIREMENTS = "requirements" JOB_PARAMETERS = "parameters" # job statuses STATUS_QUEUED = "queued" STATUS_RUNNING = "running" STATUS_SKIPPED = "skipped" STATUS_FAILED = "failed" STATUS_DONE = "done" # job definition files META_FILE = "job.yml" BASE_FILE = "base.ipynb" PARAMS_FILE = "params.yml" JOB_FILE = "job.ipynb" RESULT_FILE = "result.ipynb" # debug printing levels DEBUG_ERROR = 1 DEBUG_WARNING = 2 DEBUG_INFO = 3 # debug message functions def get_drt_imp_msg(base_class): return f"{base_class.__name__} may not be instantiated directly. " \ f"Implement a child class." def get_not_imp_msg(parent_class, class_function): return f"Children of the '{parent_class.__name__}' class must implement " \ f"the '{class_function.__name__}({signature(class_function)})' " \ "function"