summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRasmus Luha <rasmus.luha@ut.ee>2025-05-12 13:13:22 +0300
committerRasmus Luha <rasmus.luha@ut.ee>2025-05-12 13:13:22 +0300
commit6cb9f3eb1614f097992e27f2309bf6ca1cd37238 (patch)
treef4f2ca7739d98e10ed8dda201f2e088558ed9796
parentaad1cf42f85eb1a2e2a34bf997a03f2941ec99de (diff)
update README.md, cleanup
-rw-r--r--README.md8
-rw-r--r--config.py24
-rw-r--r--modules/nifi/core.py2
-rw-r--r--modules/telegraf/core.py1
4 files changed, 16 insertions, 19 deletions
diff --git a/README.md b/README.md
index 3ee758c..aa3f99c 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,12 @@
# Pipeline generator
-This is a pipeline generator that currently supports the following platforms: `Nifi`, `Telegraf`(TODO).
+This tool is a pipeline generator that currently supports the following platforms: `Nifi`, `Telegraf`.
+This was built as a part of my Bachelor's thesis.
## Usage
- `source venv/bin/activate`
- `pip install -r requirements.txt`
-- Setup sutff in `config.py` (database and nifi auth etc)
-- Then run `main.py`
-WIP
+- Setup sutff in `config.py`, for more info look at [Configuration info](#Configuration-info)
+- Then run `python main.py`
## Configuration info
diff --git a/config.py b/config.py
index 3c5ec72..26ed10a 100644
--- a/config.py
+++ b/config.py
@@ -7,27 +7,27 @@ INTERACTIVE_MODE=True
NIFI_HOST="https://127.0.0.1.nip.io"
NIFI_DEPLOY=True
-NIFI_USER="lab08nifiuser"
-NIFI_PASS="tartunifi2023"
+NIFI_USER="nifi_user"
+NIFI_PASS="nifi_passwod"
-MEASUREMENT_NAME="EurDol_kurss"
+MEASUREMENT_NAME="measurement_name"
-## Database
+
+
+##j## Database ####
DB_URL="http://influxdb:8086"
-#DB_NAME="nifi_weatherData"
-DB_NAME="nifi_valuuta"
+DB_NAME="nifi_weatherData"
DB_USER="admin"
DB_PASS="admin"
+
#### Over all ####
## Needed if Interactive mode turned off
-#API_URL="https://api.open-meteo.com/v1/forecast?latitude=37.9838&longitude=23.7275&current_weather=true"
-API_URL="https://v6.exchangerate-api.com/v6/78660310eae8ed2c9ab662f8/latest/USD"
-#API_FIELDS={'temperature': '.current_weather.temperature', 'windspeed': '.current_weather.windspeed'}
-API_FIELDS={'temp': '.main.temp', 'winds': '.wind.speed'}
-API_USERNAME="rasmus.luha"
+API_URL="https://api.open-meteo.com/v1/forecast?latitude=37.9838&longitude=23.7275&current_weather=true"
+API_FIELDS={'temperature': '.current_weather.temperature', 'windspeed': '.current_weather.windspeed'}
+API_USERNAME="Placeholder"
API_PASSWORD="Placeholder"
PIPELINE_SCHEDULING_PERIOD="10"
-PIPELINE_NAME="ExchangeRate_pipeline"
+PIPELINE_NAME="pipelineName"
diff --git a/modules/nifi/core.py b/modules/nifi/core.py
index 5c15837..0d6ca17 100644
--- a/modules/nifi/core.py
+++ b/modules/nifi/core.py
@@ -1,4 +1,3 @@
-## TODO - check syntax
from common import core as common
import config
from modules.nifi import nifi_utils
@@ -67,7 +66,6 @@ def modify_all_processors(data_values, schedulingPeriod, new_pipeline_name, api_
nifi_utils.update_template(new_pipeline_path, "flowContents.processors[3].properties", "JsonPath Expression", split_json_path)
## EvaluateJsonPath processor setup
- ## TODO
for key, value in data_values.items() :
path_parts = value.split(']')
nifi_utils.update_template(new_pipeline_path, "flowContents.processors[2].properties", key, "$"+path_parts[1])
diff --git a/modules/telegraf/core.py b/modules/telegraf/core.py
index b340742..69021a2 100644
--- a/modules/telegraf/core.py
+++ b/modules/telegraf/core.py
@@ -102,7 +102,6 @@ def build_pipeline():
measurement_name = str(input("Palun sisesta andmebaasi (influxDB) jaoks vajalik 'measurement' nimi: "))
new_pipeline_name=input("Mis saab andmekonveieri nimeks: ")+".toml"
- ## TODO
else:
api_url = config.API_URL
data_values = config.API_FIELDS