nifi json value extraction

5 files changed, 92 insertions, 18 deletions

diff --git a/README.md b/README.md
index 9706bd0..74cbb2a 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,2 @@
-TODOs
-
-API urli kontrollimine
-  - lihtsalt kontrollimine
-  - autentimine
-
-
-https://api.open-meteo.com/v1/forecast?latitude=58.38&longitude=26.72&current_weather=true
+Ekstrakti jsonist väljad, anna numbrilistina ette - dict
+    user valib numbri --> ehitad protsessori.
diff --git a/common/core.py b/common/core.py
index 44248be..5841d41 100644
--- a/common/core.py
+++ b/common/core.py
@@ -10,6 +10,24 @@ def ask_binary_input(prompt="Kas jah või ei?: ", valikud=["jah","ei"]):
             return answer
         print(f"Ebakorretne sisend.Palun vasta kas '{valikud[0]}' või '{valikud[1]}'")
 
+
+def ask_digit_input(max_index):
+    while True:
+        user_input = input(f"Vali number (0 - {max_index}): ").strip()
+
+        if not user_input.isdigit():
+            print("Palun vali korrektne numer.")
+            continue # algusesse
+
+        index = int(user_input)
+
+        if 0 <= index <= max_index:
+            return int(index)
+        else:
+            print(f" Number ei kuulu valikusse. Palun vali number vahemikus 0-{max_index}.")
+
+
+
 def is_app_url_correct(api_url, needs_auth, username,passwd):
     print("Teostan API kutset...\n")
     try:
@@ -20,21 +38,48 @@ def is_app_url_correct(api_url, needs_auth, username,passwd):
 
         response.raise_for_status() ## Check if staus code is 2xx
         data = response.json()
-        print(json.dumps(data, indent=2))
-        return True
+#        print(json.dumps(data, indent=2))
+        return data, True
 
     except requests.exceptions.RequestException as e:
         print(f"HTTP error: {e}")
-        return False
+        return None, False
     except ValueError:
         print("andmeallikas ei tagasta vallidset JSON kuju...")
-        return False
+        return None, False
     except Exception as e:
         print(f"API kutsel tekkis viga: {e}")
-        return False
+        return None, False
+
+
+## Func todo - add list level support with porcessors etc
+def inspect_json_top_level(json_data):
+    while True:
+        print(json.dumps(json_data, indent=2))
+        print("\nVali json võti millest soovid väärtuse andmekonveieriga ekstrakteerida\n")
 
+        keys = list(json_data.keys())
 
-##TODO
-def add_api_authentication():
-    print("Adding api authentication ... (TODO)")
+        for index, key in enumerate(keys):
+            value = json_data[key]
+            value_type = type(value).__name__ ## Mis type json itemgiga tegu
+            if isinstance(value, list):
+                suggestion = "SplitJson"
+            else:
+                suggestion = "EvaluateJsonPath"
+            print(f"  [{index}] {key} ({value_type})")
 
+        selected_index = ask_digit_input(len(list(json_data.keys())) - 1)
+        selected_key = keys[selected_index]
+        selected_value = json_data[selected_key]
+
+        # Wrap into new json object
+        #extracted = {selected_key: selected_value}
+
+        if isinstance(selected_value, dict) or isinstance(selected_value,list):
+            json_data = selected_value
+            continue
+        else:
+            print(f"\nValitud võti: '{selected_key}':")
+            #print(json.dumps(selected_value, indent=2))
+            return selected_key
diff --git a/config.py b/config.py
index e69de29..1274541 100644
--- a/config.py
+++ b/config.py
@@ -0,0 +1,4 @@
+## TODO
+DB=influxdb
+DB_USER=
+DB_PASS=
diff --git a/modules/nifi/core.py b/modules/nifi/core.py
index bdac035..6e2f2e7 100644
--- a/modules/nifi/core.py
+++ b/modules/nifi/core.py
@@ -3,6 +3,7 @@ from rich.console import Console
 from common import core as common
 
 import sys
+import json
 
 
 def introduction():
@@ -11,7 +12,27 @@ def introduction():
     console.print(ascii_art, style="cyan")
     print("Valisid Nifi Platformi!\n")
 
+
+## TODO
+def set_processor_property(pipeline, processor_name, property_key, property_value):
+    for processor in pipeline['flowContents']['processors']:
+        if processor['name'] == processor_name:
+            processor['properties'][property_key] = property_value
+            print(f"Updated '{property_key}' in processor '{processor_name}'")
+            return
+    print(f"Processor '{processor_name}' not found.")
+
+
+
+
+
+
+
 def build_pipeline():
+
+    chosen_json_values = []
+
+    ##Getting API url and json values
     while True:
         api_url = input("Palun sisesta andmete API URL: ").strip()
         username = "placeholder"
@@ -22,8 +43,18 @@ def build_pipeline():
             username=input("Sisesta kasutajanimi: ")
             passwd=input("Sisesta parool: ")
 
-        if common.is_app_url_correct(api_url,needs_auth,username,passwd):
-            break  # Exit loop if URL is correct
+        json_data, api_url_correct = common.is_app_url_correct(api_url,needs_auth,username,passwd)
+
+        ## TODO itemite eemaldamise v6malus
+        if api_url_correct:
+            while True:
+                chosen_json_values.append(common.inspect_json_top_level(json_data))
+                print("Oled hetkel valinud järgmised väärtused:", chosen_json_values)
+                choose_another = common.ask_binary_input(prompt="\nKas soovid (v)alida veel mõne väärtuse või liikuda (e)dasi?(v/e): ",valikud=["v","e"]).strip().lower()
+
+                if choose_another == 'e':
+                    return chosen_json_values 
+
         else:
             choice = common.ask_binary_input(prompt="\nKas soovid URL-i (m)uuta URL-i või (v)äljuda?(m/v): ",valikud=["m","v"]).strip().lower()
             if choice == 'v':
diff --git a/modules/nifi/clean.json b/modules/nifi/templates/basic_ETL.json
index 457c783..457c783 100644
--- a/modules/nifi/clean.json
+++ b/modules/nifi/templates/basic_ETL.json