-
-
Save shantanoo-desai/bca38860af1e55d0323bbad9cb89fca1 to your computer and use it in GitHub Desktop.
| - hosts: all | |
| gather_facts: false | |
| vars_files: | |
| - defaults/main.yml | |
| - vault.yml | |
| tasks: | |
| - name: Do Not Execute Playbook if No Flows File extra variable is mentioned | |
| fail: | |
| msg: > | |
| Please specify the flows file to be deployed on the fleet using: | |
| ansible-playbook deploy_flow.yml -e "flow=<name>.json" --ask-vault-pass | |
| when: flow is not defined | |
| - name: Checking if Flow File Exists in Directory | |
| stat: | |
| path: "{{flows_directory }}/{{ flow }}" | |
| delegate_to: localhost | |
| register: flow_file | |
| - name: Flow File doesn't exist | |
| fail: | |
| msg: "The Flow File does not exist in Path: {{flows_directory}}" | |
| when: not flow_file.stat.exists | |
| - name: Reading the Flow File | |
| set_fact: | |
| flow_json: "{{ lookup('ansible.builtin.file', flows_directory + '/' + flow) | from_json }}" | |
| when: flow_file.stat.exists | |
| - name: Check for Existence of either InfluxDB, MySQL or MQTT nodes in Flows | |
| set_fact: | |
| nodes_exist: true | |
| when: "flow_json | selectattr('type', 'equalto', node) | list | count > 0" | |
| loop: | |
| - influxdb | |
| - mqtt-broker | |
| - mysql | |
| loop_control: | |
| loop_var: node | |
| - name: Dynamic Credential Insertion for InfluxDB, MySQL, or MQTT Configuration Nodes | |
| # Use set_fact on `flow_json` variable because we want to update the same JSON structure | |
| set_fact: | |
| # Extract the dict and combine the credentials block in it | |
| # and merge it back into the list and update `flow_json` fact | |
| flow_json: "{{ flow_json[0:index] | |
| + [ node | combine(node_replacements[node.type]) ] | |
| + flow_json[index|int+1:] }}" | |
| # Loop over the list of dictionaries and determine the index of the dict | |
| # which has `type` value either `influxdb`, `mysql` or `mqtt` | |
| when: (nodes_exist is not undefined) and (nodes_exist is true) | |
| loop: "{{ query('ansible.utils.index_of', flow_json, 'in', matched_nodes) }}" | |
| loop_control: | |
| # Use `idx` as the variable name for the index obtained by loop | |
| loop_var: index | |
| # Observe the matches made on the Terminal as <node.id>, <node.type> for better clarity | |
| label: "{{ node.id ~ ', ' ~ node.type }}" | |
| # Variables needed for the loop | |
| vars: | |
| # call each indexed dictionary in the list as `node` | |
| node: "{{ flow_json[index] }}" | |
| # check if `type` is either `influxdb`, `mysql`, or `mqtt` and store it `matched_nodes` variable | |
| matched_nodes: "{{ flow_json | selectattr('type', 'in', node_replacements.keys()) }}" | |
| # The general Structure of the "credentials" block to be inserted if there are `matched_nodes` | |
| # The username, password credentials will be extracted from Ansible-Vault | |
| node_replacements: | |
| influxdb: | |
| credentials: | |
| username: "{{ influxdb.username }}" | |
| password: "{{ influxdb.password }}" | |
| MySQLdatabase: | |
| credentials: | |
| user: "{{ mysql.username }}" | |
| password: "{{ mysql.password }}" | |
| mqtt-broker: | |
| credentials: | |
| username: "{{ mqtt.username }}" | |
| password: "{{ mqtt.password }}" | |
| - name: Preparing Flow File for Deployment | |
| copy: | |
| dest: "{{ flows_directory + '/.' + flow }}" | |
| content: "{{ flow_json | to_nice_json }}" | |
| delegate_to: localhost | |
| - name: Obtain the Authentication Token for Node-RED Instance | |
| ansible.builtin.uri: | |
| # API: /auth/token HTTP POST | |
| # Headers: Content-Type: application/json | |
| # Body: | |
| # client_id: node-red-admin | |
| # grant_type: password | |
| # scope: * | |
| # username: Node-RED instance username | |
| # password: Node-RED instance username | |
| # Expected Response Code: 200 | |
| # Response Body: | |
| # { | |
| # "access_token": "<token>", | |
| # "expires_in": 604800, | |
| # "token_type": "Bearer" | |
| # } | |
| url: "{{ nodered_base_url }}/auth/token" | |
| method: POST | |
| body: | |
| client_id: node-red-admin | |
| grant_type: password | |
| scope: "*" | |
| username: "{{ nodered.username }}" | |
| password: "{{ nodered.password }}" | |
| body_format: form-urlencoded | |
| status_code: 200 | |
| # Store the Response in a variable called `login` | |
| register: login | |
| when: flow is defined | |
| - name: Upload Dedicated Flow on Each Node-RED Instance in Fleet | |
| ansible.builtin.uri: | |
| # API: /nodered/flows HTTP POST | |
| # Headers: | |
| # Authorization: Bearer <token> | |
| # Content-Type: application/json | |
| # Node-RED-API-Version: v1 | |
| # Node-RED-Deployment-Type: full | |
| # Body: | |
| # <Flow JSON File> | |
| # Expected Response Code: 204 | |
| # Response body: none | |
| url: "{{ nodered_base_url }}/flows" | |
| method: POST | |
| headers: | |
| # Obtain the Token and its Type from the `login` variable from previous task | |
| Authorization: "{{ login.json.token_type }} {{ login.json.access_token }}" | |
| Content-Type: application/json | |
| Node-RED-API-Version: v1 | |
| Node-RED-Deployment-Type: full | |
| # `flow` is a variable containing the <name>.json of the node-RED flow to be deployed | |
| body: "{{ lookup('ansible.builtin.file', flows_directory + '/.' + flow) }}" | |
| body_format: json | |
| status_code: 204 | |
| register: flow_upload_status | |
| when: flow is defined | |
| - name: Revoke the Authentication Token for Node-RED Instance | |
| ansible.builtin.uri: | |
| # API: /nodered/auth/revoke HTTP POST | |
| # Headers: | |
| # Authorization: Bearer <token> | |
| # Content-Type: application/json | |
| # Body: | |
| # {"token": "<token>"} | |
| # Expected Response Code: 200 | |
| # Response body: none | |
| url: "{{ nodered_base_url }}/auth/revoke" | |
| method: POST | |
| headers: | |
| # Obtain the Token and its Type from the `login` variable from previous task | |
| Authorization: "{{ login.json.token_type }} {{ login.json.access_token }}" | |
| Content-Type: application/json | |
| body: '{"token": "{{ login.json.access_token }}" }' | |
| body_format: json | |
| status_code: 200 | |
| - name: Post Deployment Cleanup | |
| file: | |
| name: "{{ flows_directory + '/.' + flow }}" | |
| state: absent | |
| when: flow_upload_status.status == 204 | |
| delegate_to: localhost |
Requirement
- The
typecould be eitherinfluxdb,mysqlormqtt - If
influxdb,mysqlormqttadd their respective credential values from ansible-vault to the JSON fact and save the file.
InfluxDB Flow with multiple Influxdb in and Influxdb out nodes but same configuration
[
{
"id": "3272da57099ccc8b",
"type": "influxdb out",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "",
"measurement": "test",
"precision": "",
"retentionPolicy": "",
"database": "",
"retentionPolicyV18Flux": "",
"org": "",
"bucket": "",
"x": 550,
"y": 220,
"wires": []
},
{
"id": "022df80929df552d",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "single value",
"func": "data = msg;\nbirthday = data.payload.Birthday;\nfirstname = data.payload.FirstName;\nlastname = data.payload.LastName;\n\nmsg.topic = `INSERT INTO Person (LastName, FirstName, Birthday) VALUES (\"${lastname}\", \"${firstname}\", \"${birthday}\")`\n\nreturn msg;\n\n",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 330,
"y": 220,
"wires": [
[
"3272da57099ccc8b",
"0c620320df54d9e9"
]
]
},
{
"id": "4b50f3cf12d784f7",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "Write into DB",
"info": "",
"x": 130,
"y": 180,
"wires": []
},
{
"id": "7e580b15a8db4ac2",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "Drop",
"info": "",
"x": 110,
"y": 480,
"wires": []
},
{
"id": "5d6402f7b51d5fb3",
"type": "inject",
"z": "ac2ce998eb52d6a1",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": "",
"topic": "",
"payload": "",
"payloadType": "date",
"x": 140,
"y": 520,
"wires": [
[
"a6ba63c5924ff968"
]
]
},
{
"id": "a6ba63c5924ff968",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "Drop DB",
"func": "msg.query=\"DROP database test;\";\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"libs": [],
"x": 320,
"y": 520,
"wires": [
[
"84cb8f6e5865614e"
]
]
},
{
"id": "84cb8f6e5865614e",
"type": "influxdb in",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "time query",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "",
"x": 530,
"y": 520,
"wires": [
[
"13b5487229b408df"
]
]
},
{
"id": "13b5487229b408df",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Drop Database",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 720,
"y": 520,
"wires": []
},
{
"id": "c3559bf5851edc50",
"type": "inject",
"z": "ac2ce998eb52d6a1",
"name": "",
"props": [
{
"p": "payload",
"v": "",
"vt": "date"
},
{
"p": "topic",
"v": "",
"vt": "string"
}
],
"repeat": "",
"crontab": "",
"once": false,
"topic": "",
"payload": "",
"payloadType": "date",
"x": 140,
"y": 400,
"wires": [
[
"94912a933da37aa3"
]
]
},
{
"id": "94912a933da37aa3",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "simple query",
"func": "msg.query=\"select * from test;\";\nreturn msg;",
"outputs": 1,
"noerr": 0,
"x": 330,
"y": 400,
"wires": [
[
"4802b98bf91f16fe"
]
]
},
{
"id": "4802b98bf91f16fe",
"type": "influxdb in",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "time query",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"x": 530,
"y": 400,
"wires": [
[
"0add337fc7a8593a"
]
]
},
{
"id": "0add337fc7a8593a",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Query DB",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 760,
"y": 400,
"wires": []
},
{
"id": "6e9584601325f77d",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "Query",
"info": "",
"x": 110,
"y": 360,
"wires": []
},
{
"id": "37341ac01c09cce8",
"type": "inject",
"z": "ac2ce998eb52d6a1",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "",
"once": true,
"onceDelay": "",
"topic": "",
"payload": "",
"payloadType": "date",
"x": 150,
"y": 120,
"wires": [
[
"117bd7d39fa81cec"
]
]
},
{
"id": "117bd7d39fa81cec",
"type": "function",
"z": "ac2ce998eb52d6a1",
"name": "create DB",
"func": "msg.query=\"CREATE database test;\";\nreturn msg;",
"outputs": 1,
"noerr": 0,
"x": 320,
"y": 120,
"wires": [
[
"295efc8813559a5e"
]
]
},
{
"id": "295efc8813559a5e",
"type": "influxdb in",
"z": "ac2ce998eb52d6a1",
"influxdb": "3b82b6ebb677a390",
"name": "time query",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "",
"x": 530,
"y": 120,
"wires": [
[
"b02d4ed96d549d8f"
]
]
},
{
"id": "b02d4ed96d549d8f",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Create DB",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 710,
"y": 120,
"wires": []
},
{
"id": "21b86b58ba40f775",
"type": "comment",
"z": "ac2ce998eb52d6a1",
"name": "CREATE",
"info": "",
"x": 120,
"y": 80,
"wires": []
},
{
"id": "0c620320df54d9e9",
"type": "debug",
"z": "ac2ce998eb52d6a1",
"name": "Write to DB",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "payload",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 530,
"y": 180,
"wires": []
},
{
"id": "0db402bfb37ce456",
"type": "ui_form",
"z": "ac2ce998eb52d6a1",
"name": "Person",
"label": "",
"group": "905b1b7c51cfbf14",
"order": 0,
"width": 0,
"height": 0,
"options": [
{
"label": "Last Name",
"value": "LastName",
"type": "text",
"required": true,
"rows": null
},
{
"label": "First Name",
"value": "FirstName",
"type": "text",
"required": true,
"rows": null
},
{
"label": "Birthday",
"value": "Birthday",
"type": "date",
"required": true,
"rows": null
}
],
"formValue": {
"LastName": "",
"FirstName": "",
"Birthday": ""
},
"payload": "",
"submit": "submit",
"cancel": "cancel",
"topic": "topic",
"topicType": "msg",
"splitLayout": false,
"className": "",
"x": 120,
"y": 220,
"wires": [
[
"022df80929df552d"
]
]
},
{
"id": "3b82b6ebb677a390",
"type": "influxdb",
"z": "ac2ce998eb52d6a1",
"hostname": "influxdb",
"port": "8086",
"database": "test",
"name": "test db",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "",
"rejectUnauthorized": false,
"credentials": {
"username": "tester",
"password": "tester",
"token": ""
}
},
{
"id": "905b1b7c51cfbf14",
"type": "ui_group",
"name": "Person",
"tab": "2b82b49d97413831",
"order": 1,
"disp": true,
"width": "6",
"collapse": false,
"className": ""
},
{
"id": "2b82b49d97413831",
"type": "ui_tab",
"name": "Person",
"icon": "dashboard",
"disabled": false,
"hidden": false
}
]jq command
each influxdb in and influxdb out node has a influxdb key in them, upon querying they all end out having the same hash value that is the id value of the node "type": "influxdb"
cat flows.json | jq '.[].influxdb'
"3b82b6ebb677a390"
null
null
null
null
null
"3b82b6ebb677a390"
null
null
null
"3b82b6ebb677a390"
null
null
null
null
"3b82b6ebb677a390"
null
null
null
null
null
null
nullSolution 1
from Kristianheljas on Ansible IRC Chat
kristianheljas 16:23:11
So, it is just looping trough indexes that match the items you find in �matched_nodes and replacing each index using list splicing to inject on into the correct place
- hosts: localhost
gather_facts: false
vars:
node_red_flows: [
{
"id": "one",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "f6f2187d.f17ca8",
"type": "tab",
"label": "Flow 1",
"disabled": false,
"info": ""
},
{
"id": "two",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "ae7554e7968c9aa1",
"type": "influxdb in",
"z": "f6f2187d.f17ca8",
"influxdb": "398670264ec6e79f",
"name": "fluxtestdb",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "organisation",
"x": 460,
"y": 300,
"wires": [
[ ]
]
}
]
tasks:
- set_fact:
node_red_flows: "{{ node_red_flows[0:item] + [node_red_flows[item] | combine(combine_with)] + node_red_flows[item|int+1:] }}"
loop: "{{ query('ansible.utils.index_of', node_red_flows, 'in', matched_nodes) }}"
vars:
matched_nodes: "{{ node_red_flows | selectattr('type', '==', 'influxdb') }}"
combine_with:
credentials:
username: tester
password: tester
- debug:
msg: "{{ node_red_flows }}"Solution
by kristian heljas
explanation
I added loop_control as well to provide nicer output for the item and more meaningul name to the index (opposed to item)
[4:58:27 PM] Using the label option, you can configure howok: [localhost] => (item=****THIS*****)looks like in the ansible output
[4:59:12 PM] And �loop_var allows you to change the default variable name �item, which in this case might have been confusing
- hosts: localhost
gather_facts: false
vars:
node_red_flows: [
{
"id": "one",
"type": "mysql",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "f6f2187d.f17ca8",
"type": "tab",
"label": "Flow 1",
"disabled": false,
"info": ""
},
{
"id": "two",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
},
{
"id": "ae7554e7968c9aa1",
"type": "influxdb in",
"z": "f6f2187d.f17ca8",
"influxdb": "398670264ec6e79f",
"name": "fluxtestdb",
"query": "",
"rawOutput": false,
"precision": "",
"retentionPolicy": "",
"org": "organisation",
"x": 460,
"y": 300,
"wires": [
[ ]
]
},
{
"id": "three",
"type": "influxdb",
"hostname": "127.0.0.1",
"port": "8086",
"protocol": "http",
"database": "test",
"name": "testfluxdb",
"usetls": false,
"tls": "",
"influxdbVersion": "1.x",
"url": "http://localhost:8086",
"rejectUnauthorized": true
}
]
tasks:
- set_fact:
node_red_flows: "{{ node_red_flows[0:index] + [node | combine(node_replacements[node.type])] + node_red_flows[index|int+1:] }}"
loop: "{{ query('ansible.utils.index_of', node_red_flows, 'in', matched_nodes) }}"
loop_control:
loop_var: index
label: "{{ node.id ~ ', ' ~ node.type }}"
vars:
node: "{{ node_red_flows[index] }}"
matched_nodes: "{{ node_red_flows | selectattr('type', 'in', node_replacements.keys()) }}"
node_replacements:
influxdb:
credentials:
username: influxdb-tester
password: influxdb-tester
mysql:
credentials:
username: mysql-tester
password: mysql-tester
- debug:
msg: "{{ node_red_flows }}"
Source