-
-
Save shantanoo-desai/bca38860af1e55d0323bbad9cb89fca1 to your computer and use it in GitHub Desktop.
- hosts: all | |
gather_facts: false | |
vars_files: | |
- defaults/main.yml | |
- vault.yml | |
tasks: | |
- name: Do Not Execute Playbook if No Flows File extra variable is mentioned | |
fail: | |
msg: > | |
Please specify the flows file to be deployed on the fleet using: | |
ansible-playbook deploy_flow.yml -e "flow=<name>.json" --ask-vault-pass | |
when: flow is not defined | |
- name: Checking if Flow File Exists in Directory | |
stat: | |
path: "{{flows_directory }}/{{ flow }}" | |
delegate_to: localhost | |
register: flow_file | |
- name: Flow File doesn't exist | |
fail: | |
msg: "The Flow File does not exist in Path: {{flows_directory}}" | |
when: not flow_file.stat.exists | |
- name: Reading the Flow File | |
set_fact: | |
flow_json: "{{ lookup('ansible.builtin.file', flows_directory + '/' + flow) | from_json }}" | |
when: flow_file.stat.exists | |
- name: Check for Existence of either InfluxDB, MySQL or MQTT nodes in Flows | |
set_fact: | |
nodes_exist: true | |
when: "flow_json | selectattr('type', 'equalto', node) | list | count > 0" | |
loop: | |
- influxdb | |
- mqtt-broker | |
- mysql | |
loop_control: | |
loop_var: node | |
- name: Dynamic Credential Insertion for InfluxDB, MySQL, or MQTT Configuration Nodes | |
# Use set_fact on `flow_json` variable because we want to update the same JSON structure | |
set_fact: | |
# Extract the dict and combine the credentials block in it | |
# and merge it back into the list and update `flow_json` fact | |
flow_json: "{{ flow_json[0:index] | |
+ [ node | combine(node_replacements[node.type]) ] | |
+ flow_json[index|int+1:] }}" | |
# Loop over the list of dictionaries and determine the index of the dict | |
# which has `type` value either `influxdb`, `mysql` or `mqtt` | |
when: (nodes_exist is not undefined) and (nodes_exist is true) | |
loop: "{{ query('ansible.utils.index_of', flow_json, 'in', matched_nodes) }}" | |
loop_control: | |
# Use `idx` as the variable name for the index obtained by loop | |
loop_var: index | |
# Observe the matches made on the Terminal as <node.id>, <node.type> for better clarity | |
label: "{{ node.id ~ ', ' ~ node.type }}" | |
# Variables needed for the loop | |
vars: | |
# call each indexed dictionary in the list as `node` | |
node: "{{ flow_json[index] }}" | |
# check if `type` is either `influxdb`, `mysql`, or `mqtt` and store it `matched_nodes` variable | |
matched_nodes: "{{ flow_json | selectattr('type', 'in', node_replacements.keys()) }}" | |
# The general Structure of the "credentials" block to be inserted if there are `matched_nodes` | |
# The username, password credentials will be extracted from Ansible-Vault | |
node_replacements: | |
influxdb: | |
credentials: | |
username: "{{ influxdb.username }}" | |
password: "{{ influxdb.password }}" | |
MySQLdatabase: | |
credentials: | |
user: "{{ mysql.username }}" | |
password: "{{ mysql.password }}" | |
mqtt-broker: | |
credentials: | |
username: "{{ mqtt.username }}" | |
password: "{{ mqtt.password }}" | |
- name: Preparing Flow File for Deployment | |
copy: | |
dest: "{{ flows_directory + '/.' + flow }}" | |
content: "{{ flow_json | to_nice_json }}" | |
delegate_to: localhost | |
- name: Obtain the Authentication Token for Node-RED Instance | |
ansible.builtin.uri: | |
# API: /auth/token HTTP POST | |
# Headers: Content-Type: application/json | |
# Body: | |
# client_id: node-red-admin | |
# grant_type: password | |
# scope: * | |
# username: Node-RED instance username | |
# password: Node-RED instance username | |
# Expected Response Code: 200 | |
# Response Body: | |
# { | |
# "access_token": "<token>", | |
# "expires_in": 604800, | |
# "token_type": "Bearer" | |
# } | |
url: "{{ nodered_base_url }}/auth/token" | |
method: POST | |
body: | |
client_id: node-red-admin | |
grant_type: password | |
scope: "*" | |
username: "{{ nodered.username }}" | |
password: "{{ nodered.password }}" | |
body_format: form-urlencoded | |
status_code: 200 | |
# Store the Response in a variable called `login` | |
register: login | |
when: flow is defined | |
- name: Upload Dedicated Flow on Each Node-RED Instance in Fleet | |
ansible.builtin.uri: | |
# API: /nodered/flows HTTP POST | |
# Headers: | |
# Authorization: Bearer <token> | |
# Content-Type: application/json | |
# Node-RED-API-Version: v1 | |
# Node-RED-Deployment-Type: full | |
# Body: | |
# <Flow JSON File> | |
# Expected Response Code: 204 | |
# Response body: none | |
url: "{{ nodered_base_url }}/flows" | |
method: POST | |
headers: | |
# Obtain the Token and its Type from the `login` variable from previous task | |
Authorization: "{{ login.json.token_type }} {{ login.json.access_token }}" | |
Content-Type: application/json | |
Node-RED-API-Version: v1 | |
Node-RED-Deployment-Type: full | |
# `flow` is a variable containing the <name>.json of the node-RED flow to be deployed | |
body: "{{ lookup('ansible.builtin.file', flows_directory + '/.' + flow) }}" | |
body_format: json | |
status_code: 204 | |
register: flow_upload_status | |
when: flow is defined | |
- name: Revoke the Authentication Token for Node-RED Instance | |
ansible.builtin.uri: | |
# API: /nodered/auth/revoke HTTP POST | |
# Headers: | |
# Authorization: Bearer <token> | |
# Content-Type: application/json | |
# Body: | |
# {"token": "<token>"} | |
# Expected Response Code: 200 | |
# Response body: none | |
url: "{{ nodered_base_url }}/auth/revoke" | |
method: POST | |
headers: | |
# Obtain the Token and its Type from the `login` variable from previous task | |
Authorization: "{{ login.json.token_type }} {{ login.json.access_token }}" | |
Content-Type: application/json | |
body: '{"token": "{{ login.json.access_token }}" }' | |
body_format: json | |
status_code: 200 | |
- name: Post Deployment Cleanup | |
file: | |
name: "{{ flows_directory + '/.' + flow }}" | |
state: absent | |
when: flow_upload_status.status == 204 | |
delegate_to: localhost |
Requirement
- The
type
could be eitherinfluxdb
,mysql
ormqtt
- If
influxdb
,mysql
ormqtt
add their respective credential values from ansible-vault to the JSON fact and save the file.
InfluxDB Flow with multiple Influxdb in and Influxdb out nodes but same configuration
[
{
"id": "3272da57099ccc8b",
"type": "influxdb out",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "",
"measurement": "test",
"precision": "",
"retentionPolicy": "",
"database": "",
"retentionPolicyV18Flux": "",
"org": "",
"bucket": "",
"x": 550,
"y": 220,
"wires": []
},
{
"id": "022df80929df552d",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "single value",
"func": "data = msg;\nbirthday = data.payload.Birthday;\nfirstname = data.payload.FirstName;\nlastname = data.payload.LastName;\n\nmsg.topic = `INSERT INTO Person (LastName, FirstName, Birthday) VALUES (\"${lastname}\", \"${firstname}\", \"${birthday}\")`\n\nreturn msg;\n\n",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 330,
"y": 220,
"wires": [
[
"3272da57099ccc8b",
"0c620320df54d9e9"
]
]
},
{
"id": "4b50f3cf12d784f7",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "Write into DB",
"info": "",
"x": 130,
"y": 180,
"wires": []
},
{
"id": "7e580b15a8db4ac2",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "Drop",
"info": "",
"x": 110,
"y": 480,
"wires": []
},
{
"id": "5d6402f7b51d5fb3",
"type": "inject",
"z": "ac2ce998eb52d6a1",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": "",
"topic": "",
"payload": "",
"payloadType": "date",
"x": 140,
"y": 520,
"wires": [
[
"a6ba63c5924ff968"
]
]
},
{
"id": "a6ba63c5924ff968",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "Drop DB",
"func": "msg.query=\"DROP database test;\";\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 320,
"y": 520,
"wires": [
[
"84cb8f6e5865614e"
]
]
},
{
"id": "84cb8f6e5865614e",
"type": "influxdb in",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "time query",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "",
"x": 530,
"y": 520,
"wires": [
[
"13b5487229b408df"
]
]
},
{
"id": "13b5487229b408df",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Drop Database",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 720,
"y": 520,
"wires": []
},
{
"id": "c3559bf5851edc50",
"type": "inject",
"z": "ac2ce998eb52d6a1",
"name": "",
"props": [
{
"p": "payload",
"v": "",
"vt": "date"
},
{
"p": "topic",
"v": "",
"vt": "string"
}
],
"repeat": "",
"crontab": "",
"once": false,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 140,
"y": 400,
"wires": [
[
"94912a933da37aa3"
]
]
},
{
"id": "94912a933da37aa3",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "simple query",
"func": "msg.query=\"select * from test;\";\nreturn msg;",
"outputs": 1,
"noerr": 0,
"x": 330,
"y": 400,
"wires": [
[
"4802b98bf91f16fe"
]
]
},
{
"id": "4802b98bf91f16fe",
"type": "influxdb in",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "time query",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"x": 530,
"y": 400,
"wires": [
[
"0add337fc7a8593a"
]
]
},
{
"id": "0add337fc7a8593a",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Query DB",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 760,
"y": 400,
"wires": []
},
{
"id": "6e9584601325f77d",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "Query",
"info": "",
"x": 110,
"y": 360,
"wires": []
},
{
"id": "37341ac01c09cce8",
"type": "inject",
"z": "ac2ce998eb52d6a1",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "",
"once": true,
"onceDelay": "",
"topic": "",
"payload": "",
"payloadType": "date",
"x": 150,
"y": 120,
"wires": [
[
"117bd7d39fa81cec"
]
]
},
{
"id": "117bd7d39fa81cec",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "create DB",
"func": "msg.query=\"CREATE database test;\";\nreturn msg;",
"outputs": 1,
"noerr": 0,
"x": 320,
"y": 120,
"wires": [
[
"295efc8813559a5e"
]
]
},
{
"id": "295efc8813559a5e",
"type": "influxdb in",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "time query",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "",
"x": 530,
"y": 120,
"wires": [
[
"b02d4ed96d549d8f"
]
]
},
{
"id": "b02d4ed96d549d8f",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Create DB",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 710,
"y": 120,
"wires": []
},
{
"id": "21b86b58ba40f775",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "CREATE",
"info": "",
"x": 120,
"y": 80,
"wires": []
},
{
"id": "0c620320df54d9e9",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Write to DB",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 530,
"y": 180,
"wires": []
},
{
"id": "0db402bfb37ce456",
"type": "ui_form",
"z": "ac2ce998eb52d6a1",
"name": "Person",
"label": "",
"group": "905b1b7c51cfbf14",
"order": 0,
"width": 0,
"height": 0,
"options": [
{
"label": "Last Name",
"value": "LastName",
"type": "text",
"required": true,
"rows": null
},
{
"label": "First Name",
"value": "FirstName",
"type": "text",
"required": true,
"rows": null
},
{
"label": "Birthday",
"value": "Birthday",
"type": "date",
"required": true,
"rows": null
}
],
"formValue": {
"LastName": "",
"FirstName": "",
"Birthday": ""
},
"payload": "",
"submit": "submit",
"cancel": "cancel",
"topic": "topic",
"topicType": "msg",
"splitLayout": false,
"className": "",
"x": 120,
"y": 220,
"wires": [
[
"022df80929df552d"
]
]
},
{
"id": "3b82b6ebb677a390",
"type": "influxdb",
"z": "ac2ce998eb52d6a1",
"hostname": "influxdb",
"port": "8086",
"database": "test",
"name": "test db",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "",
"rejectUnauthorized": false,
"credentials": {
"username": "tester",
"password": "tester",
"token": ""
}
},
{
"id": "905b1b7c51cfbf14",
"type": "ui_group",
"name": "Person",
"tab": "2b82b49d97413831",
"order": 1,
"disp": true,
"width": "6",
"collapse": false,
"className": ""
},
{
"id": "2b82b49d97413831",
"type": "ui_tab",
"name": "Person",
"icon": "dashboard",
"disabled": false,
"hidden": false
}
]
jq
command
each influxdb in
and influxdb out
node has a influxdb
key in them, upon querying they all end out having the same hash value that is the id
value of the node "type": "influxdb"
cat flows.json | jq '.[].influxdb'
"3b82b6ebb677a390"
null
null
null
null
null
"3b82b6ebb677a390"
null
null
null
"3b82b6ebb677a390"
null
null
null
null
"3b82b6ebb677a390"
null
null
null
null
null
null
null
Solution 1
from Kristianheljas on Ansible IRC Chat
kristianheljas 16:23:11
So, it is just looping trough indexes that match the items you find in �matched_nodes and replacing each index using list splicing to inject on into the correct place
- hosts: localhost
gather_facts: false
vars:
node_red_flows: [
{
"id": "one",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "f6f2187d.f17ca8",
"type": "tab",
"label": "Flow 1",
"disabled": false,
"info": ""
},
{
"id": "two",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "ae7554e7968c9aa1",
"type": "influxdb in",
"z": "f6f2187d.f17ca8",
"influxdb": "398670264ec6e79f",
"name": "fluxtestdb",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "organisation",
"x": 460,
"y": 300,
"wires": [
[ ]
]
}
]
tasks:
- set_fact:
node_red_flows: "{{ node_red_flows[0:item] + [node_red_flows[item] | combine(combine_with)] + node_red_flows[item|int+1:] }}"
loop: "{{ query('ansible.utils.index_of', node_red_flows, 'in', matched_nodes) }}"
vars:
matched_nodes: "{{ node_red_flows | selectattr('type', '==', 'influxdb') }}"
combine_with:
credentials:
username: tester
password: tester
- debug:
msg: "{{ node_red_flows }}"
Solution
by kristian heljas
explanation
I added loop_control as well to provide nicer output for the item and more meaningul name to the index (opposed to item)
[4:58:27 PM] Using the label option, you can configure howok: [localhost] => (item=****THIS*****)
looks like in the ansible output
[4:59:12 PM] And �loop_var allows you to change the default variable name �item, which in this case might have been confusing
- hosts: localhost
gather_facts: false
vars:
node_red_flows: [
{
"id": "one",
"type": "mysql",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "f6f2187d.f17ca8",
"type": "tab",
"label": "Flow 1",
"disabled": false,
"info": ""
},
{
"id": "two",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "ae7554e7968c9aa1",
"type": "influxdb in",
"z": "f6f2187d.f17ca8",
"influxdb": "398670264ec6e79f",
"name": "fluxtestdb",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "organisation",
"x": 460,
"y": 300,
"wires": [
[ ]
]
},
{
"id": "three",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
}
]
tasks:
- set_fact:
node_red_flows: "{{ node_red_flows[0:index] + [node | combine(node_replacements[node.type])] + node_red_flows[index|int+1:] }}"
loop: "{{ query('ansible.utils.index_of', node_red_flows, 'in', matched_nodes) }}"
loop_control:
loop_var: index
label: "{{ node.id ~ ', ' ~ node.type }}"
vars:
node: "{{ node_red_flows[index] }}"
matched_nodes: "{{ node_red_flows | selectattr('type', 'in', node_replacements.keys()) }}"
node_replacements:
influxdb:
credentials:
username: influxdb-tester
password: influxdb-tester
mysql:
credentials:
username: mysql-tester
password: mysql-tester
- debug:
msg: "{{ node_red_flows }}"
Source