diff options
| -rw-r--r-- | README.md | 10 | ||||
| -rw-r--r-- | common/core.py | 61 | ||||
| -rw-r--r-- | config.py | 4 | ||||
| -rw-r--r-- | modules/nifi/core.py | 35 | ||||
| -rw-r--r-- | modules/nifi/templates/basic_ETL.json (renamed from modules/nifi/clean.json) | 0 |
5 files changed, 92 insertions, 18 deletions
@@ -1,8 +1,2 @@ -TODOs - -API urli kontrollimine - - lihtsalt kontrollimine - - autentimine - - -https://api.open-meteo.com/v1/forecast?latitude=58.38&longitude=26.72¤t_weather=true +Ekstrakti jsonist väljad, anna numbrilistina ette - dict + user valib numbri --> ehitad protsessori. diff --git a/common/core.py b/common/core.py index 44248be..5841d41 100644 --- a/common/core.py +++ b/common/core.py @@ -10,6 +10,24 @@ def ask_binary_input(prompt="Kas jah või ei?: ", valikud=["jah","ei"]): return answer print(f"Ebakorretne sisend.Palun vasta kas '{valikud[0]}' või '{valikud[1]}'") + +def ask_digit_input(max_index): + while True: + user_input = input(f"Vali number (0 - {max_index}): ").strip() + + if not user_input.isdigit(): + print("Palun vali korrektne numer.") + continue # algusesse + + index = int(user_input) + + if 0 <= index <= max_index: + return int(index) + else: + print(f" Number ei kuulu valikusse. Palun vali number vahemikus 0-{max_index}.") + + + def is_app_url_correct(api_url, needs_auth, username,passwd): print("Teostan API kutset...\n") try: @@ -20,21 +38,48 @@ def is_app_url_correct(api_url, needs_auth, username,passwd): response.raise_for_status() ## Check if staus code is 2xx data = response.json() - print(json.dumps(data, indent=2)) - return True +# print(json.dumps(data, indent=2)) + return data, True except requests.exceptions.RequestException as e: print(f"HTTP error: {e}") - return False + return None, False except ValueError: print("andmeallikas ei tagasta vallidset JSON kuju...") - return False + return None, False except Exception as e: print(f"API kutsel tekkis viga: {e}") - return False + return None, False + + +## Func todo - add list level support with porcessors etc +def inspect_json_top_level(json_data): + while True: + print(json.dumps(json_data, indent=2)) + print("\nVali json võti millest soovid väärtuse andmekonveieriga ekstrakteerida\n") + keys = list(json_data.keys()) -##TODO -def add_api_authentication(): - print("Adding api authentication ... (TODO)") + for index, key in enumerate(keys): + value = json_data[key] + value_type = type(value).__name__ ## Mis type json itemgiga tegu + if isinstance(value, list): + suggestion = "SplitJson" + else: + suggestion = "EvaluateJsonPath" + print(f" [{index}] {key} ({value_type})") + selected_index = ask_digit_input(len(list(json_data.keys())) - 1) + selected_key = keys[selected_index] + selected_value = json_data[selected_key] + + # Wrap into new json object + #extracted = {selected_key: selected_value} + + if isinstance(selected_value, dict) or isinstance(selected_value,list): + json_data = selected_value + continue + else: + print(f"\nValitud võti: '{selected_key}':") + #print(json.dumps(selected_value, indent=2)) + return selected_key @@ -0,0 +1,4 @@ +## TODO +DB=influxdb +DB_USER= +DB_PASS= diff --git a/modules/nifi/core.py b/modules/nifi/core.py index bdac035..6e2f2e7 100644 --- a/modules/nifi/core.py +++ b/modules/nifi/core.py @@ -3,6 +3,7 @@ from rich.console import Console from common import core as common import sys +import json def introduction(): @@ -11,7 +12,27 @@ def introduction(): console.print(ascii_art, style="cyan") print("Valisid Nifi Platformi!\n") + +## TODO +def set_processor_property(pipeline, processor_name, property_key, property_value): + for processor in pipeline['flowContents']['processors']: + if processor['name'] == processor_name: + processor['properties'][property_key] = property_value + print(f"Updated '{property_key}' in processor '{processor_name}'") + return + print(f"Processor '{processor_name}' not found.") + + + + + + + def build_pipeline(): + + chosen_json_values = [] + + ##Getting API url and json values while True: api_url = input("Palun sisesta andmete API URL: ").strip() username = "placeholder" @@ -22,8 +43,18 @@ def build_pipeline(): username=input("Sisesta kasutajanimi: ") passwd=input("Sisesta parool: ") - if common.is_app_url_correct(api_url,needs_auth,username,passwd): - break # Exit loop if URL is correct + json_data, api_url_correct = common.is_app_url_correct(api_url,needs_auth,username,passwd) + + ## TODO itemite eemaldamise v6malus + if api_url_correct: + while True: + chosen_json_values.append(common.inspect_json_top_level(json_data)) + print("Oled hetkel valinud järgmised väärtused:", chosen_json_values) + choose_another = common.ask_binary_input(prompt="\nKas soovid (v)alida veel mõne väärtuse või liikuda (e)dasi?(v/e): ",valikud=["v","e"]).strip().lower() + + if choose_another == 'e': + return chosen_json_values + else: choice = common.ask_binary_input(prompt="\nKas soovid URL-i (m)uuta URL-i või (v)äljuda?(m/v): ",valikud=["m","v"]).strip().lower() if choice == 'v': diff --git a/modules/nifi/clean.json b/modules/nifi/templates/basic_ETL.json index 457c783..457c783 100644 --- a/modules/nifi/clean.json +++ b/modules/nifi/templates/basic_ETL.json |