NikolajDanger/meow_base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2 from toppggg/Windows1

Browse Source 
Windows
This commit is contained in:
Patch
2023-02-08 15:32:15 +01:00
committed by GitHub
parent d787e37adc ca10fd6dc7
commit a5191e351b
2 changed files with 15 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/functionality.py
View File

@ -8,7 +8,7 @@ import yaml
from datetime import datetime from datetime import datetime
from multiprocessing.connection import Connection, wait as multi_wait from multiprocessing.connection import Connection, PipeConnection, wait as multi_wait
from multiprocessing.queues import Queue from multiprocessing.queues import Queue
from papermill.translators import papermill_translators from papermill.translators import papermill_translators
from typing import Any from typing import Any
@ -49,11 +49,12 @@ def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]: def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]:
all_connections = [i for i in inputs if type(i) is Connection] \ all_connections = [i for i in inputs if type(i) is Connection] \
+ [i for i in inputs if type(i) is PipeConnection] \
+ [i._reader for i in inputs if type(i) is Queue] + [i._reader for i in inputs if type(i) is Queue]
ready = multi_wait(all_connections) ready = multi_wait(all_connections)
ready_inputs = [i for i in inputs if \ ready_inputs = [i for i in inputs if \
(type(i) is Connection and i in ready) \ (type(i) is Connection and i in ready) \
or (type(i) is PipeConnection and i in ready) \
or (type(i) is Queue and i._reader in ready)] or (type(i) is Queue and i._reader in ready)]
return ready_inputs return ready_inputs

24
tests/test_runner.py
View File

@ -121,10 +121,10 @@ class MeowTests(unittest.TestCase):
# Test single meow papermill job execution # Test single meow papermill job execution
def testMeowRunnerPapermillExecution(self)->None: def testMeowRunnerPapermillExecution(self)->None:
pattern_one = FileEventPattern( pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile", "pattern_one", os.path.join("start","A.txt"), "recipe_one", "infile",
parameters={ parameters={
"extra":"A line from a test Pattern", "extra":"A line from a test Pattern",
"outfile":"{VGRID}/output/{FILENAME}" "outfile":os.path.join("{VGRID}","output","{FILENAME}")
}) })
recipe = JupyterNotebookRecipe( recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK) "recipe_one", APPENDING_NOTEBOOK)
@ -205,16 +205,16 @@ class MeowTests(unittest.TestCase):
# Test meow papermill job chaining within runner # Test meow papermill job chaining within runner
def testMeowRunnerLinkedPapermillExecution(self)->None: def testMeowRunnerLinkedPapermillExecution(self)->None:
pattern_one = FileEventPattern( pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile", "pattern_one", os.path.join("start","A.txt"), "recipe_one", "infile",
parameters={ parameters={
"extra":"A line from Pattern 1", "extra":"A line from Pattern 1",
"outfile":"{VGRID}/middle/{FILENAME}" "outfile":os.path.join("{VGRID}","middle","{FILENAME}")
}) })
pattern_two = FileEventPattern( pattern_two = FileEventPattern(
"pattern_two", "middle/A.txt", "recipe_one", "infile", "pattern_two", os.path.join("middle","A.txt"), "recipe_one", "infile",
parameters={ parameters={
"extra":"A line from Pattern 2", "extra":"A line from Pattern 2",
"outfile":"{VGRID}/output/{FILENAME}" "outfile":os.path.join("{VGRID}","output","{FILENAME}")
}) })
recipe = JupyterNotebookRecipe( recipe = JupyterNotebookRecipe(
"recipe_one", APPENDING_NOTEBOOK) "recipe_one", APPENDING_NOTEBOOK)
@ -314,10 +314,10 @@ class MeowTests(unittest.TestCase):
# Test single meow python job execution # Test single meow python job execution
def testMeowRunnerPythonExecution(self)->None: def testMeowRunnerPythonExecution(self)->None:
pattern_one = FileEventPattern( pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile", "pattern_one", os.path.join("start","A.txt"), "recipe_one", "infile",
parameters={ parameters={
"num":10000, "num":10000,
"outfile":"{VGRID}/output/{FILENAME}" "outfile":os.path.join("{VGRID}","output","{FILENAME}")
}) })
recipe = PythonRecipe( recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT "recipe_one", COMPLETE_PYTHON_SCRIPT
@ -403,16 +403,16 @@ class MeowTests(unittest.TestCase):
# Test meow python job chaining within runner # Test meow python job chaining within runner
def testMeowRunnerLinkedPythonExecution(self)->None: def testMeowRunnerLinkedPythonExecution(self)->None:
pattern_one = FileEventPattern( pattern_one = FileEventPattern(
"pattern_one", "start/A.txt", "recipe_one", "infile", "pattern_one", os.path.join("start","A.txt"), "recipe_one", "infile",
parameters={ parameters={
"num":250, "num":250,
"outfile":"{VGRID}/middle/{FILENAME}" "outfile":os.path.join("{VGRID}","middle","{FILENAME}")
}) })
pattern_two = FileEventPattern( pattern_two = FileEventPattern(
"pattern_two", "middle/A.txt", "recipe_one", "infile", "pattern_two", os.path.join("middle","A.txt"), "recipe_one", "infile",
parameters={ parameters={
"num":40, "num":40,
"outfile":"{VGRID}/output/{FILENAME}" "outfile":os.path.join("{VGRID}","output","{FILENAME}")
}) })
recipe = PythonRecipe( recipe = PythonRecipe(
"recipe_one", COMPLETE_PYTHON_SCRIPT "recipe_one", COMPLETE_PYTHON_SCRIPT