diff options
| -rw-r--r-- | README.md | 26 | ||||
| -rw-r--r-- | common/core.py | 1 | ||||
| -rw-r--r-- | config.py | 2 | ||||
| -rw-r--r-- | modules/telegraf/testing.py | 17 | ||||
| -rw-r--r-- | requirements.txt | 1 |
5 files changed, 15 insertions, 32 deletions
@@ -14,17 +14,6 @@ Configuration variables are defined in `config.py`, where sample variables are a WIP ### Variables - -#### Nifi - -| Variable Name | Description | Example Value | -|---------------------|----------------------------------------------|----------------------------| -| `NIFI_HOST` | Base URL of the Apache NiFi instance | `https://127.0.0.1.nip.io` | -| `NIFI_USER` | Username for authenticating in Nifi | `nifi_username` | -| `NIFI_PASS` | Username for authenticating in Nifi | `nifi_password` | -| `NIFI_DEPLOY` | Automatically delpoy pipeline to Nifi | `True` | -| `INTERACTIVE_MODE` | If False you can parse through api call data | `True` | - In interactive mode you can manually give the API url, credentials and data fields for the pipeline from API response If you opt out the interactive mode you have to define needed variables in the config.py file before using the tool. @@ -34,8 +23,8 @@ The following variables have to be defined if interactive mode is tured to `Fals |-------------------------------|-------------------------------------------|-----------------------------------------------------------------------------------------------| | API_URL | API url that returens JSON data | `https://api.open-meteo.com/v1/forecast?latitude=58.38&longitude=26.72¤t_weather=true"` | | API_FIELDS | Data fields for pipeline, with json paths | `{'temperature': '.current_weather.temperature', 'windspeed': '.current_weather.windspeed'}` | -| API_USERNAME | Api username, if required | `"Placeholder"` | -| API_PASSWORD | Api username, if required | `"Placeholder"` | +| API_USERNAME | Api username, if required by API | `"Placeholder"` | +| API_PASSWORD | Api username, if required by API | `"Placeholder"` | | PIPELINE_SCHEDULING_PERIOD | How often the pipeline should run | `"5 sec"` | | PIPELINE_NAME | Name of the pipeline | `"test_pipeline.json"` | @@ -50,6 +39,17 @@ Database variales have to be filled out before using the tool as currently there | `DB_PASS` | Database passwod | `admin` | +#### Nifi +Here Nifi Host, Username and Password are needed for deploying the pipeline to the Nifi Host. + +| Variable Name | Description | Example Value | +|---------------------|----------------------------------------------|----------------------------| +| `NIFI_HOST` | Base URL of the Apache NiFi instance | `https://127.0.0.1.nip.io` | +| `NIFI_USER` | Username for authenticating in Nifi | `nifi_username` | +| `NIFI_PASS` | Username for authenticating in Nifi | `nifi_password` | +| `NIFI_DEPLOY` | Automatically delpoy pipeline to Nifi | `True` | +| `INTERACTIVE_MODE` | If False you can parse through api call data | `True` | + ## Current Repo structure ``` diff --git a/common/core.py b/common/core.py index f47a8b5..85d8657 100644 --- a/common/core.py +++ b/common/core.py @@ -55,7 +55,6 @@ def is_app_url_correct(api_url, needs_auth, username,passwd): return None, False -## TODO - add list level support with porcessors etc ## Asks the user json item to extract and returns it as dict item-value pair, where item is name and value json path def inspect_json_top_level(json_data): """ @@ -8,7 +8,7 @@ NIFI_HOST="https://127.0.0.1.nip.io" NIFI_DEPLOY=True NIFI_USER="nifi_user" -NIFI_PASS="nifi_passwod" +NIFI_PASS="nifi_pass" MEASUREMENT_NAME="measurement_name" diff --git a/modules/telegraf/testing.py b/modules/telegraf/testing.py deleted file mode 100644 index d049bbf..0000000 --- a/modules/telegraf/testing.py +++ /dev/null @@ -1,17 +0,0 @@ -import toml - -def modify_input(template, new_pipeline_path, key, value): - data = toml.load(template) - pluggin = data["inputs"]["http"][0] - - if key in pluggin: - - print(f"Before: {key} = {http_input[key]}") - http_input[key] = value - print(f"After: {key} = {http_input[key]}") - - - with open(new_pipeline_path, "w") as f: - toml.dump(data, f) - -modify_input("templates/basic_ETL.toml", "test_pipers.toml, "urls", ["stillTesting"]) diff --git a/requirements.txt b/requirements.txt index c16f892..7ae9633 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ pyfiglet==1.0.2 Pygments==2.19.1 requests==2.32.3 rich==14.0.0 +toml==0.10.2 urllib3==2.3.0 |