updated code so should work on windows, with exception of waiting on multiple connections

This commit is contained in:
PatchOfScotland
2023-02-08 14:46:03 +01:00
parent d787e37adc
commit a1451881ae
17 changed files with 280 additions and 205 deletions

View File

@ -7,11 +7,12 @@ import os
import yaml
from datetime import datetime
from typing import List
from multiprocessing.connection import Connection, wait as multi_wait
from multiprocessing.queues import Queue
from papermill.translators import papermill_translators
from typing import Any
from typing import Any, Dict
from random import SystemRandom
from core.correctness.validation import check_type, valid_existing_file_path, \
@ -36,7 +37,7 @@ KEYWORD_JOB = "{JOB}"
#TODO Make this guaranteed unique
def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
def generate_id(prefix:str="", length:int=16, existing_ids:List[str]=[],
charset:str=CHAR_UPPERCASE+CHAR_LOWERCASE, attempts:int=24):
random_length = max(length - len(prefix), 0)
for _ in range(attempts):
@ -47,10 +48,12 @@ def generate_id(prefix:str="", length:int=16, existing_ids:list[str]=[],
raise ValueError(f"Could not generate ID unique from '{existing_ids}' "
f"using values '{charset}' and length of '{length}'.")
def wait(inputs:list[VALID_CHANNELS])->list[VALID_CHANNELS]:
def wait(inputs:List[VALID_CHANNELS])->List[VALID_CHANNELS]:
all_connections = [i for i in inputs if type(i) is Connection] \
+ [i._reader for i in inputs if type(i) is Queue]
for i in inputs:
print(type(i))
ready = multi_wait(all_connections)
ready_inputs = [i for i in inputs if \
(type(i) is Connection and i in ready) \
@ -72,7 +75,6 @@ def _get_file_sha256(file_path):
def get_file_hash(file_path:str, hash:str):
check_type(hash, str)
import os
valid_existing_file_path(file_path)
valid_hashes = {
@ -80,7 +82,7 @@ def get_file_hash(file_path:str, hash:str):
}
if hash not in valid_hashes:
raise KeyError(f"Cannot use hash '{hash}'. Valid are "
"'{list(valid_hashes.keys())}")
f"'{list(valid_hashes.keys())}")
return valid_hashes[hash](file_path)
@ -169,7 +171,7 @@ def read_notebook(filepath:str):
with open(filepath, 'r') as read_file:
return json.load(read_file)
def write_notebook(source:dict[str,Any], filename:str):
def write_notebook(source:Dict[str,Any], filename:str):
"""
Writes the given notebook source code to a given filename.
@ -183,10 +185,10 @@ def write_notebook(source:dict[str,Any], filename:str):
json.dump(source, job_file)
# Adapted from: https://github.com/rasmunk/notebook_parameterizer
def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any],
parameters:dict[str,Any], expand_env_values:bool=False)->dict[str,Any]:
def parameterize_jupyter_notebook(jupyter_notebook:Dict[str,Any],
parameters:Dict[str,Any], expand_env_values:bool=False)->Dict[str,Any]:
nbformat.validate(jupyter_notebook)
check_type(parameters, dict)
check_type(parameters, Dict)
if jupyter_notebook["nbformat"] != 4:
raise Warning(
@ -256,10 +258,10 @@ def parameterize_jupyter_notebook(jupyter_notebook:dict[str,Any],
return output_notebook
def parameterize_python_script(script:list[str], parameters:dict[str,Any],
expand_env_values:bool=False)->dict[str,Any]:
def parameterize_python_script(script:List[str], parameters:Dict[str,Any],
expand_env_values:bool=False)->Dict[str,Any]:
check_script(script)
check_type(parameters, dict)
check_type(parameters, Dict)
output_script = copy.deepcopy(script)
@ -302,8 +304,8 @@ def print_debug(print_target, debug_level, msg, level)->None:
status = "WARNING"
print(f"{status}: {msg}", file=print_target)
def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
monitor_base:str)->dict[str,str]:
def replace_keywords(old_dict:Dict[str,str], job_id:str, src_path:str,
monitor_base:str)->Dict[str,str]:
"""Function to replace all MEOW magic words in a dictionary with dynamic
values."""
new_dict = {}
@ -332,8 +334,8 @@ def replace_keywords(old_dict:dict[str,str], job_id:str, src_path:str,
return new_dict
def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={}
)->dict[Any,Any]:
def create_event(event_type:str, path:str, rule:Any, extras:Dict[Any,Any]={}
)->Dict[Any,Any]:
"""Function to create a MEOW dictionary."""
return {
**extras,
@ -343,7 +345,7 @@ def create_event(event_type:str, path:str, rule:Any, extras:dict[Any,Any]={}
}
def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
extras:dict[Any,Any]={})->dict[Any,Any]:
extras:Dict[Any,Any]={})->Dict[Any,Any]:
"""Function to create a MEOW event dictionary."""
return create_event(
EVENT_TYPE_WATCHDOG,
@ -358,8 +360,8 @@ def create_watchdog_event(path:str, rule:Any, base:str, hash:str,
}
)
def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
)->dict[Any,Any]:
def create_job(job_type:str, event:Dict[str,Any], extras:Dict[Any,Any]={}
)->Dict[Any,Any]:
"""Function to create a MEOW job dictionary."""
job_dict = {
#TODO compress event?
@ -376,7 +378,7 @@ def create_job(job_type:str, event:dict[str,Any], extras:dict[Any,Any]={}
return {**extras, **job_dict}
def lines_to_string(lines:list[str])->str:
def lines_to_string(lines:List[str])->str:
"""Function to convert a list of str lines, into one continuous string
separated by newline characters"""
return "\n".join(lines)