From 6b3c096f6efea8855772c7faffa90f71861290bd Mon Sep 17 00:00:00 2001 From: Rasmus Luha Date: Sat, 26 Apr 2025 01:10:52 +0300 Subject: telegraf initial pluggin modifiying implemented --- config.py | 2 +- modules/nifi/core.py | 2 +- modules/telegraf/core.py | 71 +++++++++++++++++++++++++++++++ modules/telegraf/telegraf_utils.py | 17 ++++++++ modules/telegraf/templates/basic_ETL.toml | 6 +-- modules/telegraf/testing.py | 17 ++++++++ 6 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 modules/telegraf/telegraf_utils.py create mode 100644 modules/telegraf/testing.py diff --git a/config.py b/config.py index 3b1f304..684349c 100644 --- a/config.py +++ b/config.py @@ -23,7 +23,7 @@ API_FIELDS={'temperature': '.current_weather.temperature', 'windspeed': '.curren API_USERNAME="Placeholder" API_PASSWORD="Placehoder" PIPELINE_SCHEDULING_PERIOD="5 sec" -PIPELINE_NAME="test_pipeline.json" +PIPELINE_NAME="test_pipeline" diff --git a/modules/nifi/core.py b/modules/nifi/core.py index 46455c7..118ee3a 100644 --- a/modules/nifi/core.py +++ b/modules/nifi/core.py @@ -103,7 +103,7 @@ def build_pipeline(): api_url = config.API_URL data_values = config.API_FIELDS schedulingPeriod = config.PIPELINE_SCHEDULING_PERIOD - new_pipeline_name = config.PIPELINE_NAME + new_pipeline_name = config.PIPELINE_NAME+".json" api_username = config.API_USERNAME api_password = config.API_PASSWORD diff --git a/modules/telegraf/core.py b/modules/telegraf/core.py index 1d49917..3febf7b 100644 --- a/modules/telegraf/core.py +++ b/modules/telegraf/core.py @@ -2,9 +2,80 @@ from pyfiglet import figlet_format from rich.console import Console +from modules.telegraf import telegraf_utils +#from modules.nifi import nifi_utils +from common import core as common +import config + + +import toml +import shutil + + + def introduction(): console = Console() ascii_art = figlet_format("Telegraf") console.print(ascii_art, style="cyan") print("Valisid Telegraf Platformi!\n") + + + +########################### + + +def modify_template(new_pipeline_path, api_url): + #telegraf_utils.modify_output("templates/basic_ETL.toml", "urls", "testingIfWorks") + + ## Api Url editing + telegraf_utils.modify_input(new_pipeline_path,"urls", [api_url]) + + + + + + +########################### + + + +def build_pipeline(): + if config.INTERACTIVE_MODE: + data_values, api_url, api_username, api_password= common.get_data_values() + + print("\nKui tihti peaks andmekonveier jooksma? (sekundites)") + schedulingPeriod = str(common.ask_digit_input(86400))+ "sec" + + new_pipeline_name=input("Mis saab andmekonveieri nimeks: ")+".toml" + + ## TODO + else: + api_url = config.API_URL + data_values = config.API_FIELDS + schedulingPeriod = config.PIPELINE_SCHEDULING_PERIOD + new_pipeline_name = config.PIPELINE_NAME+".toml" + api_username = config.API_USERNAME + api_password = config.API_PASSWORD + + + + + ### Select template + ##TODO + template_name="basic_ETL.toml" + + new_pipeline_path = f"pipelines/{new_pipeline_name}" + shutil.copy(f"modules/telegraf/templates/{template_name}", new_pipeline_path) + + + modify_template(new_pipeline_path, api_url) + #telegraf.modify_output("templates/basic_ETL.toml", "urls", "testingIfWorks") + + + + + + + print("end currently") + #print(f"✅✅✅ Valmis. Uus genereeritud andmekoveier nimega '{new_pipeline_name}' asub kaustas 'pipelines'.") diff --git a/modules/telegraf/telegraf_utils.py b/modules/telegraf/telegraf_utils.py new file mode 100644 index 0000000..d6ed376 --- /dev/null +++ b/modules/telegraf/telegraf_utils.py @@ -0,0 +1,17 @@ +import toml + +def modify_input(new_pipeline_path, key, value): + data = toml.load(new_pipeline_path) + pluggin = data["inputs"]["http"][0] + + if key in pluggin: + + print(f"Before: {key} = {pluggin[key]}") + pluggin[key] = value + print(f"After: {key} = {pluggin[key]}") + + + with open(new_pipeline_path, "w") as f: + toml.dump(data, f) + +#modify_input("templates/basic_ETL.toml", "test_pipers.toml, "urls", ["stillTesting"]) diff --git a/modules/telegraf/templates/basic_ETL.toml b/modules/telegraf/templates/basic_ETL.toml index 02806c2..155f570 100644 --- a/modules/telegraf/templates/basic_ETL.toml +++ b/modules/telegraf/templates/basic_ETL.toml @@ -7,15 +7,15 @@ [[inputs.http]] urls = [] #[ "https://api.open-meteo.com/v1/forecast?latitude=58.38&longitude=26.72¤t_weather=true" ] method = "GET" - timeout = #"5s" + timeout = "plcaeholder" #"5s" headers = { Content-Type = "application/json" } data_format = "json" - json_query = #"current_weather" + json_query = "plcaeholder" #"current_weather" fieldinclude = [] #["temperature", "windspeed"] #tag_keys = ["temperature", "windspeed"] # Measuremens for DB - name_override = "weather_metrics" + name_override = "plcaeholder" #"weather_metrics" # Output Plugin: InfluxDB diff --git a/modules/telegraf/testing.py b/modules/telegraf/testing.py new file mode 100644 index 0000000..d049bbf --- /dev/null +++ b/modules/telegraf/testing.py @@ -0,0 +1,17 @@ +import toml + +def modify_input(template, new_pipeline_path, key, value): + data = toml.load(template) + pluggin = data["inputs"]["http"][0] + + if key in pluggin: + + print(f"Before: {key} = {http_input[key]}") + http_input[key] = value + print(f"After: {key} = {http_input[key]}") + + + with open(new_pipeline_path, "w") as f: + toml.dump(data, f) + +modify_input("templates/basic_ETL.toml", "test_pipers.toml, "urls", ["stillTesting"]) -- cgit v1.2.3