NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added watchdog file monitoring

Browse Source
This commit is contained in:
PatchOfScotland
2022-12-13 14:59:43 +01:00
parent 4041b86343
commit 380f7066e1
13 changed files with 550 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
core/correctness/__init__.py
View File

@ -0,0 +1,3 @@
from core.correctness.validation import *
from core.correctness.vars import *

17
core/correctness/validation.py
View File

@ -1,7 +1,9 @@
from abc import ABCMeta
from os.path import sep
from typing import Any, _SpecialForm
from core.correctness.vars import VALID_PATH_CHARS
def check_input(variable:Any, expected_type:type, alt_types:list[type]=[],
or_none:bool=False)->None:
"""
@ -73,8 +75,8 @@ def valid_dict(variable:dict[Any, Any], key_type:type, value_type:type,
required_keys:list[Any]=[], optional_keys:list[Any]=[],
strict:bool=True, min_length:int=1)->None:
check_input(variable, dict)
check_input(key_type, type, alt_types=[_SpecialForm, ABCMeta])
check_input(value_type, type, alt_types=[_SpecialForm, ABCMeta])
check_input(key_type, type, alt_types=[_SpecialForm])
check_input(value_type, type, alt_types=[_SpecialForm])
check_input(required_keys, list)
check_input(optional_keys, list)
check_input(strict, bool)
@ -110,3 +112,12 @@ def valid_list(variable:list[Any], entry_type:type,
f"of length {min_length}")
for entry in variable:
check_input(entry, entry_type, alt_types=alt_types)
def valid_path(variable:str, allow_base=False, extension:str="", min_length=1):
valid_string(variable, VALID_PATH_CHARS, min_length=min_length)
if not allow_base and variable.startswith(sep):
raise ValueError(f"Cannot accept path '{variable}'. Must be relative.")
if min_length > 0 and extension and not variable.endswith(extension):
raise ValueError(f"Path '{variable}' does not have required "
f"extension '{extension}'.")

48
core/correctness/vars.py
View File

@ -1,6 +1,8 @@
import os
from inspect import signature
CHAR_LOWERCASE = 'abcdefghijklmnopqrstuvwxyz'
CHAR_UPPERCASE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
CHAR_NUMERIC = '0123456789'
@ -15,4 +17,48 @@ VALID_VARIABLE_NAME_CHARS = CHAR_UPPERCASE + CHAR_LOWERCASE + CHAR_NUMERIC + "_"
VALID_JUPYTER_NOTEBOOK_FILENAME_CHARS = VALID_NAME_CHARS + "." + os.path.sep
VALID_JUPYTER_NOTEBOOK_EXTENSIONS = [".ipynb"]
VALID_TRIGGERING_PATH_CHARS = VALID_NAME_CHARS + "." + os.path.sep
VALID_PATH_CHARS = VALID_NAME_CHARS + "." + os.path.sep
VALID_TRIGGERING_PATH_CHARS = VALID_NAME_CHARS + ".*" + os.path.sep
BAREBONES_NOTEBOOK = {
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 4
}
FILE_CREATE_EVENT = "file_created"
FILE_MODIFY_EVENT = "file_modified"
FILE_MOVED_EVENT = "file_moved"
FILE_CLOSED_EVENT = "file_closed"
FILE_DELETED_EVENT = "file_deleted"
FILE_EVENTS = [
FILE_CREATE_EVENT,
FILE_MODIFY_EVENT,
FILE_MOVED_EVENT,
FILE_CLOSED_EVENT,
FILE_DELETED_EVENT
]
DIR_CREATE_EVENT = "dir_created"
DIR_MODIFY_EVENT = "dir_modified"
DIR_MOVED_EVENT = "dir_moved"
DIR_DELETED_EVENT = "dir_deleted"
DIR_EVENTS = [
DIR_CREATE_EVENT,
DIR_MODIFY_EVENT,
DIR_MOVED_EVENT,
DIR_DELETED_EVENT
]
PIPE_READ = 0
PIPE_WRITE = 1
def get_drt_imp_msg(base_class):
return f"{base_class.__name__} may not be instantiated directly. " \
f"Implement a child class."
def get_not_imp_msg(parent_class, class_function):
return f"Children of the '{parent_class.__name__}' class must implement " \
f"the '{class_function.__name__}({signature(class_function)})' " \
"function"

142
core/meow.py
View File

@ -1,11 +1,13 @@
import core.correctness.vars
import core.correctness.validation
from abc import ABC, abstractmethod
from multiprocessing.connection import Connection
from typing import Any
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_PATTERN_NAME_CHARS, VALID_RULE_NAME_CHARS, \
get_not_imp_msg, get_drt_imp_msg
from core.correctness.validation import valid_string
class BaseRecipe:
name:str
recipe:Any
@ -13,6 +15,15 @@ class BaseRecipe:
requirements:dict[str, Any]
def __init__(self, name:str, recipe:Any, parameters:dict[str,Any]={},
requirements:dict[str,Any]={}):
if (type(self)._is_valid_recipe == BaseRecipe._is_valid_recipe):
msg = get_not_imp_msg(BaseRecipe, BaseRecipe._is_valid_recipe)
raise NotImplementedError(msg)
if (type(self)._is_valid_parameters == BaseRecipe._is_valid_parameters):
msg = get_not_imp_msg(BaseRecipe, BaseRecipe._is_valid_parameters)
raise NotImplementedError(msg)
if (type(self)._is_valid_requirements == BaseRecipe._is_valid_requirements):
msg = get_not_imp_msg(BaseRecipe, BaseRecipe._is_valid_requirements)
raise NotImplementedError(msg)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
@ -24,33 +35,39 @@ class BaseRecipe:
def __new__(cls, *args, **kwargs):
if cls is BaseRecipe:
raise TypeError("BaseRecipe may not be instantiated directly")
msg = get_drt_imp_msg(BaseRecipe)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
core.correctness.validation.valid_string(
name, core.correctness.vars.VALID_RECIPE_NAME_CHARS)
valid_string(name, VALID_RECIPE_NAME_CHARS)
@abstractmethod
def _is_valid_recipe(self, recipe:Any)->None:
pass
@abstractmethod
def _is_valid_parameters(self, parameters:Any)->None:
pass
@abstractmethod
def _is_valid_requirements(self, requirements:Any)->None:
pass
class BasePattern(ABC):
class BasePattern:
name:str
recipe:str
parameters:dict[str, Any]
outputs:dict[str, Any]
def __init__(self, name:str, recipe:str, parameters:dict[str,Any]={},
outputs:dict[str,Any]={}):
if (type(self)._is_valid_recipe == BasePattern._is_valid_recipe):
msg = get_not_imp_msg(BasePattern, BasePattern._is_valid_recipe)
raise NotImplementedError(msg)
if (type(self)._is_valid_parameters == BasePattern._is_valid_parameters):
msg = get_not_imp_msg(BasePattern, BasePattern._is_valid_parameters)
raise NotImplementedError(msg)
if (type(self)._is_valid_output == BasePattern._is_valid_output):
msg = get_not_imp_msg(BasePattern, BasePattern._is_valid_output)
raise NotImplementedError(msg)
self._is_valid_name(name)
self.name = name
self._is_valid_recipe(recipe)
@ -62,33 +79,37 @@ class BasePattern(ABC):
def __new__(cls, *args, **kwargs):
if cls is BasePattern:
raise TypeError("BasePattern may not be instantiated directly")
msg = get_drt_imp_msg(BasePattern)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
core.correctness.validation.valid_string(
name, core.correctness.vars.VALID_PATTERN_NAME_CHARS)
valid_string(name, VALID_PATTERN_NAME_CHARS)
@abstractmethod
def _is_valid_recipe(self, recipe:Any)->None:
pass
@abstractmethod
def _is_valid_parameters(self, parameters:Any)->None:
pass
@abstractmethod
def _is_valid_output(self, outputs:Any)->None:
pass
class BaseRule(ABC):
class BaseRule:
name:str
pattern:BasePattern
recipe:BaseRecipe
pattern_type:str=""
recipe_type:str=""
def __init__(self, name:str, pattern:BasePattern, recipe:BaseRecipe):
if (type(self)._is_valid_pattern == BaseRule._is_valid_pattern):
msg = get_not_imp_msg(BaseRule, BaseRule._is_valid_pattern)
raise NotImplementedError(msg)
if (type(self)._is_valid_recipe == BaseRule._is_valid_recipe):
msg = get_not_imp_msg(BaseRule, BaseRule._is_valid_recipe)
raise NotImplementedError(msg)
self._is_valid_name(name)
self.name = name
self._is_valid_pattern(pattern)
@ -99,18 +120,16 @@ class BaseRule(ABC):
def __new__(cls, *args, **kwargs):
if cls is BaseRule:
raise TypeError("BaseRule may not be instantiated directly")
msg = get_drt_imp_msg(BaseRule)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_name(self, name:str)->None:
core.correctness.validation.valid_string(
name, core.correctness.vars.VALID_RULE_NAME_CHARS)
valid_string(name, VALID_RULE_NAME_CHARS)
@abstractmethod
def _is_valid_pattern(self, pattern:Any)->None:
pass
@abstractmethod
def _is_valid_recipe(self, recipe:Any)->None:
pass
@ -121,3 +140,78 @@ class BaseRule(ABC):
if self.recipe_type == "":
raise AttributeError(f"Rule Class '{self.__class__.__name__}' "
"does not set a recipe_type.")
class BaseMonitor:
rules: dict[str, BaseRule]
report: Connection
listen: Connection
def __init__(self, rules:dict[str, BaseRule], report:Connection,
listen:Connection) -> None:
if (type(self).start == BaseMonitor.start):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor.start)
raise NotImplementedError(msg)
if (type(self).stop == BaseMonitor.stop):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor.stop)
raise NotImplementedError(msg)
if (type(self)._is_valid_report == BaseMonitor._is_valid_report):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor._is_valid_report)
raise NotImplementedError(msg)
self._is_valid_report(report)
self.report = report
if (type(self)._is_valid_listen == BaseMonitor._is_valid_listen):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor._is_valid_listen)
raise NotImplementedError(msg)
self._is_valid_listen(listen)
self.listen = listen
if (type(self)._is_valid_rules == BaseMonitor._is_valid_rules):
msg = get_not_imp_msg(BaseMonitor, BaseMonitor._is_valid_rules)
raise NotImplementedError(msg)
self._is_valid_rules(rules)
self.rules = rules
def __new__(cls, *args, **kwargs):
if cls is BaseMonitor:
msg = get_drt_imp_msg(BaseMonitor)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_report(self, report:Connection)->None:
pass
def _is_valid_listen(self, listen:Connection)->None:
pass
def _is_valid_rules(self, rules:dict[str, BaseRule])->None:
pass
def start(self)->None:
pass
def stop(self)->None:
pass
class BaseHandler:
inputs:Any
def __init__(self, inputs:Any) -> None:
if (type(self).handle == BaseHandler.handle):
msg = get_not_imp_msg(BaseHandler, BaseHandler.handle)
raise NotImplementedError(msg)
if (type(self)._is_valid_inputs == BaseHandler._is_valid_inputs):
msg = get_not_imp_msg(BaseHandler, BaseHandler._is_valid_inputs)
raise NotImplementedError(msg)
self._is_valid_inputs(inputs)
self.inputs = inputs
def __new__(cls, *args, **kwargs):
if cls is BaseHandler:
msg = get_drt_imp_msg(BaseHandler)
raise TypeError(msg)
return object.__new__(cls)
def _is_valid_inputs(self, inputs:Any)->None:
pass
def handle()->None:
pass

2
patterns/__init__.py
View File

@ -1,2 +1,2 @@
from patterns.file_event_pattern import FileEventPattern
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor

193
patterns/file_event_pattern.py
View File

@ -1,32 +1,62 @@
import threading
import os
from fnmatch import translate
from multiprocessing.connection import Connection
from re import match
from time import time, sleep
from typing import Any
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler, FileCreatedEvent, \
FileModifiedEvent, FileMovedEvent, FileClosedEvent, FileDeletedEvent, \
DirCreatedEvent, DirDeletedEvent, DirModifiedEvent, DirMovedEvent
from core.correctness.validation import check_input, valid_string, valid_dict
from core.correctness.validation import check_input, valid_string, \
valid_dict, valid_list, valid_path
from core.correctness.vars import VALID_RECIPE_NAME_CHARS, \
VALID_VARIABLE_NAME_CHARS
from core.meow import BasePattern
VALID_VARIABLE_NAME_CHARS, FILE_EVENTS, FILE_CREATE_EVENT, \
FILE_MODIFY_EVENT, FILE_MOVED_EVENT, FILE_CLOSED_EVENT, \
FILE_DELETED_EVENT, DIR_CREATE_EVENT, DIR_DELETED_EVENT, \
DIR_MODIFY_EVENT, DIR_MOVED_EVENT
from core.meow import BasePattern, BaseMonitor, BaseRule
_EVENT_TRANSLATIONS = {
FileCreatedEvent: FILE_CREATE_EVENT,
FileModifiedEvent: FILE_MODIFY_EVENT,
FileMovedEvent: FILE_MOVED_EVENT,
FileClosedEvent: FILE_CLOSED_EVENT,
FileDeletedEvent: FILE_DELETED_EVENT,
DirCreatedEvent: DIR_CREATE_EVENT,
DirDeletedEvent: DIR_DELETED_EVENT,
DirModifiedEvent: DIR_MODIFY_EVENT,
DirMovedEvent: DIR_MOVED_EVENT
}
class FileEventPattern(BasePattern):
triggering_path:str
triggering_file:str
event_mask:list[str]
def __init__(self, name:str, triggering_path:str, recipe:str,
triggering_file:str, parameters:dict[str,Any]={},
outputs:dict[str,Any]={}):
triggering_file:str, event_mask:list[str]=FILE_EVENTS,
parameters:dict[str,Any]={}, outputs:dict[str,Any]={}):
super().__init__(name, recipe, parameters, outputs)
self._is_valid_triggering_path(triggering_path)
self.triggering_path = triggering_path
self._is_valid_triggering_file(triggering_file)
self.triggering_file = triggering_file
self._is_valid_event_mask(event_mask)
self.event_mask = event_mask
def _is_valid_recipe(self, recipe:str)->None:
valid_string(recipe, VALID_RECIPE_NAME_CHARS)
def _is_valid_triggering_path(self, triggering_path:str)->None:
check_input(triggering_path, str)
valid_path(triggering_path)
if len(triggering_path) < 1:
raise ValueError (
f"trigginering path '{triggering_path}' is too short. "
f"triggiering path '{triggering_path}' is too short. "
"Minimum length is 1"
)
@ -42,3 +72,152 @@ class FileEventPattern(BasePattern):
valid_dict(outputs, str, str, strict=False, min_length=0)
for k in outputs.keys():
valid_string(k, VALID_VARIABLE_NAME_CHARS)
def _is_valid_event_mask(self, event_mask)->None:
valid_list(event_mask, str, min_length=1)
for mask in event_mask:
if mask not in FILE_EVENTS:
raise ValueError(f"Invalid event mask '{mask}'. Valid are: "
f"{FILE_EVENTS}")
class WatchdogMonitor(BaseMonitor):
event_handler:PatternMatchingEventHandler
monitor:Observer
base_dir:str
_rules_lock:threading.Lock
def __init__(self, base_dir:str, rules:dict[str, BaseRule],
report:Connection, listen:Connection, autostart=False,
settletime:int=1)->None:
super().__init__(rules, report, listen)
self._is_valid_base_dir(base_dir)
self.base_dir = base_dir
check_input(settletime, int)
self._rules_lock = threading.Lock()
self.event_handler = MEOWEventHandler(self, settletime=settletime)
self.monitor = Observer()
self.monitor.schedule(
self.event_handler,
self.base_dir,
recursive=True
)
if autostart:
self.start()
def start(self)->None:
self.monitor.start()
def stop(self)->None:
self.monitor.stop()
def match(self, event)->None:
src_path = event.src_path
event_type = "dir_"+ event.event_type if event.is_directory \
else "file_" + event.event_type
handle_path = src_path.replace(self.base_dir, '', 1)
while handle_path.startswith(os.path.sep):
handle_path = handle_path[1:]
self._rules_lock.acquire()
try:
for rule in self.rules.values():
if event_type not in rule.pattern.event_mask:
continue
target_path = rule.pattern.triggering_path
recursive_regexp = translate(target_path)
direct_regexp = recursive_regexp.replace('.*', '[^/]*')
recursive_hit = match(recursive_regexp, handle_path)
direct_hit = match(direct_regexp, handle_path)
if direct_hit or recursive_hit:
self.report.send((event, rule))
except Exception as e:
self._rules_lock.release()
raise Exception(e)
self._rules_lock.release()
def _is_valid_base_dir(self, base_dir:str)->None:
valid_path(base_dir)
def _is_valid_report(self, report:Connection)->None:
check_input(report, Connection)
def _is_valid_listen(self, listen:Connection)->None:
check_input(listen, Connection)
def _is_valid_rules(self, rules:dict[str, BaseRule])->None:
valid_dict(rules, str, BaseRule, min_length=0, strict=False)
class MEOWEventHandler(PatternMatchingEventHandler):
monitor:WatchdogMonitor
_settletime:int
_recent_jobs:dict[str, Any]
_recent_jobs_lock:threading.Lock
def __init__(self, monitor:WatchdogMonitor, settletime:int=1):
super().__init__()
self.monitor = monitor
self._settletime = settletime
self._recent_jobs = {}
self._recent_jobs_lock = threading.Lock()
def threaded_handler(self, event):
self._recent_jobs_lock.acquire()
try:
if event.src_path in self._recent_jobs:
recent_timestamp = self._recent_jobs[event.src_path]
difference = event.time_stamp - recent_timestamp
if difference <= self._settletime:
self._recent_jobs[event.src_path] = \
max(recent_timestamp, event.time_stamp)
self._recent_jobs_lock.release()
return
else:
self._recent_jobs[event.src_path] = event.time_stamp
else:
self._recent_jobs[event.src_path] = event.time_stamp
except Exception as ex:
self._recent_jobs_lock.release()
raise Exception(ex)
self._recent_jobs_lock.release()
self.monitor.match(event)
def handle_event(self, event):
event.time_stamp = time()
waiting_for_threaded_resources = True
while waiting_for_threaded_resources:
try:
worker = threading.Thread(
target=self.threaded_handler,
args=[event])
worker.daemon = True
worker.start()
waiting_for_threaded_resources = False
except threading.ThreadError:
sleep(1)
def on_created(self, event):
self.handle_event(event)
def on_modified(self, event):
self.handle_event(event)
def on_moved(self, event):
self.handle_event(event)
def on_deleted(self, event):
self.handle_event(event)
def on_closed(self, event):
self.handle_event(event)

6
recipes/jupyter_notebook_recipe.py
View File

@ -3,7 +3,8 @@ import nbformat
from typing import Any
from core.correctness.validation import check_input, valid_string, valid_dict
from core.correctness.validation import check_input, valid_string, \
valid_dict, valid_path
from core.correctness.vars import VALID_JUPYTER_NOTEBOOK_FILENAME_CHARS, \
VALID_JUPYTER_NOTEBOOK_EXTENSIONS, VALID_VARIABLE_NAME_CHARS
from core.meow import BaseRecipe
@ -17,8 +18,7 @@ class JupyterNotebookRecipe(BaseRecipe):
self.source = source
def _is_valid_source(self, source:str)->None:
valid_string(
source, VALID_JUPYTER_NOTEBOOK_FILENAME_CHARS, min_length=0)
valid_path(source, extension=".ipynb", min_length=0)
if not source:
return

3
rules/file_event_jupyter_notebook_rule.py
View File

@ -20,6 +20,3 @@ class FileEventJupyterNotebookRule(BaseRule):
def _is_valid_recipe(self, recipe:JupyterNotebookRecipe) -> None:
check_input(recipe, JupyterNotebookRecipe)
def _set_pattern_type(self)->None:
self.pattern_type

10
tests/test_functionality.py
View File

@ -1,19 +1,13 @@
import unittest
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE
from core.correctness.vars import CHAR_LOWERCASE, CHAR_UPPERCASE, \
BAREBONES_NOTEBOOK
from core.functionality import create_rules, generate_id
from core.meow import BaseRule
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
BAREBONES_NOTEBOOK = {
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 4
}
valid_pattern_one = FileEventPattern(
"pattern_one", "path_one", "recipe_one", "file_one")
valid_pattern_two = FileEventPattern(

97
tests/test_meow.py
View File

@ -1,7 +1,11 @@
import unittest
from core.meow import BasePattern, BaseRecipe, BaseRule
from typing import Any
from core.correctness.vars import BAREBONES_NOTEBOOK
from core.meow import BasePattern, BaseRecipe, BaseRule, BaseMonitor, \
BaseHandler
class MeowTests(unittest.TestCase):
@ -13,12 +17,97 @@ class MeowTests(unittest.TestCase):
def testBaseRecipe(self)->None:
with self.assertRaises(TypeError):
BaseRecipe("", "")
BaseRecipe("name", "")
class NewRecipe(BaseRecipe):
pass
with self.assertRaises(NotImplementedError):
NewRecipe("name", "")
class FullRecipe(BaseRecipe):
def _is_valid_recipe(self, recipe:Any)->None:
pass
def _is_valid_parameters(self, parameters:Any)->None:
pass
def _is_valid_requirements(self, requirements:Any)->None:
pass
FullRecipe("name", "")
def testBasePattern(self)->None:
with self.assertRaises(TypeError):
BasePattern("", "")
BasePattern("name", "", "", "")
class NewPattern(BasePattern):
pass
with self.assertRaises(NotImplementedError):
NewPattern("name", "", "", "")
class FullPattern(BasePattern):
def _is_valid_recipe(self, recipe:Any)->None:
pass
def _is_valid_parameters(self, parameters:Any)->None:
pass
def _is_valid_output(self, outputs:Any)->None:
pass
FullPattern("name", "", "", "")
def testBaseRule(self)->None:
with self.assertRaises(TypeError):
BaseRule("", "")
BaseRule("name", "", "")
class NewRule(BaseRule):
pass
with self.assertRaises(NotImplementedError):
NewRule("name", "", "")
class FullRule(BaseRule):
pattern_type = "pattern"
recipe_type = "recipe"
def _is_valid_recipe(self, recipe:Any)->None:
pass
def _is_valid_pattern(self, pattern:Any)->None:
pass
FullRule("name", "", "")
def testBaseMonitor(self)->None:
with self.assertRaises(TypeError):
BaseMonitor("", "", "")
class TestMonitor(BaseMonitor):
pass
with self.assertRaises(NotImplementedError):
TestMonitor("", "", "")
class FullTestMonitor(BaseMonitor):
def start(self):
pass
def stop(self):
pass
def _is_valid_report(self, report:Any)->None:
pass
def _is_valid_listen(self, listen:Any)->None:
pass
def _is_valid_rules(self, rules:Any)->None:
pass
FullTestMonitor("", "", "")
def testBaseHandler(self)->None:
with self.assertRaises(TypeError):
BaseHandler("")
class TestHandler(BaseHandler):
pass
with self.assertRaises(NotImplementedError):
TestHandler("")
class FullTestHandler(BaseHandler):
def handle(self):
pass
def _is_valid_inputs(self, inputs:Any)->None:
pass
FullTestHandler("")

84
tests/test_patterns.py
View File

@ -1,15 +1,28 @@
import shutil
import os
import unittest
from patterns.file_event_pattern import FileEventPattern
from multiprocessing import Pipe
from core.correctness.vars import FILE_EVENTS, FILE_CREATE_EVENT, PIPE_READ, \
PIPE_WRITE, BAREBONES_NOTEBOOK
from core.functionality import create_rules
from patterns.file_event_pattern import FileEventPattern, WatchdogMonitor
from recipes import JupyterNotebookRecipe
TEST_BASE = "test_base"
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None:
return super().setUp()
super().setUp()
if not os.path.exists(TEST_BASE):
os.mkdir(TEST_BASE)
def tearDown(self) -> None:
return super().tearDown()
super().tearDown()
if os.path.exists(TEST_BASE):
shutil.rmtree(TEST_BASE)
def testFileEventPatternCreationMinimum(self)->None:
FileEventPattern("name", "path", "recipe", "file")
@ -79,3 +92,68 @@ class CorrectnessTests(unittest.TestCase):
fep = FileEventPattern(
"name", "path", "recipe", "file", outputs=outputs)
self.assertEqual(fep.outputs, outputs)
def testFileEventPatternEventMask(self)->None:
fep = FileEventPattern("name", "path", "recipe", "file")
self.assertEqual(fep.event_mask, FILE_EVENTS)
with self.assertRaises(TypeError):
fep = FileEventPattern("name", "path", "recipe", "file",
event_mask=FILE_CREATE_EVENT)
with self.assertRaises(ValueError):
fep = FileEventPattern("name", "path", "recipe", "file",
event_mask=["nope"])
with self.assertRaises(ValueError):
fep = FileEventPattern("name", "path", "recipe", "file",
event_mask=[FILE_CREATE_EVENT, "nope"])
self.assertEqual(fep.event_mask, FILE_EVENTS)
def testWatchdogMonitorMinimum(self)->None:
to_monitor = Pipe()
from_monitor = Pipe()
WatchdogMonitor(TEST_BASE, {}, from_monitor[PIPE_WRITE],
to_monitor[PIPE_READ])
def testWatchdogMonitorEventIdentificaion(self)->None:
to_monitor = Pipe()
from_monitor = Pipe()
pattern_one = FileEventPattern(
"pattern_one", "A", "recipe_one", "file_one")
recipe = JupyterNotebookRecipe(
"recipe_one", BAREBONES_NOTEBOOK)
patterns = {
pattern_one.name: pattern_one,
}
recipes = {
recipe.name: recipe,
}
rules = create_rules(patterns, recipes)
wm = WatchdogMonitor(TEST_BASE, rules, from_monitor[PIPE_WRITE],
to_monitor[PIPE_READ])
wm.start()
open(os.path.join(TEST_BASE, "A"), "w")
if from_monitor[PIPE_READ].poll(3):
message = from_monitor[PIPE_READ].recv()
self.assertIsNotNone(message)
event, rule = message
self.assertIsNotNone(event)
self.assertIsNotNone(rule)
self.assertEqual(event.src_path, os.path.join(TEST_BASE, "A"))
open(os.path.join(TEST_BASE, "B"), "w")
if from_monitor[PIPE_READ].poll(3):
new_message = from_monitor[PIPE_READ].recv()
else:
new_message = None
self.assertIsNone(new_message)
wm.stop()

8
tests/test_recipes.py
View File

@ -3,13 +3,7 @@ import jsonschema
import unittest
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
BAREBONES_NOTEBOOK = {
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 4
}
from core.correctness.vars import BAREBONES_NOTEBOOK
class CorrectnessTests(unittest.TestCase):
def setUp(self)->None:

2
tests/test_rules.py
View File

@ -1,10 +1,10 @@
import unittest
from core.correctness.vars import BAREBONES_NOTEBOOK
from patterns.file_event_pattern import FileEventPattern
from recipes.jupyter_notebook_recipe import JupyterNotebookRecipe
from rules.file_event_jupyter_notebook_rule import FileEventJupyterNotebookRule
from test_recipes import BAREBONES_NOTEBOOK
class CorrectnessTests(unittest.TestCase):
def setUp(self) -> None: