IoT historical data retrieval

[oprince]

Due to IoT enhancement of historical data management, my node-red flow should now retrieve IoT events historical data from Cloudant (not directly from IoT as before). I use 'Cloudant in' node to retrieve that data, and it works fine. As the database name changes over time according to the IoT bucket interval, database name has to be set dynamically.

Would it be possible to enable dynamic database name at node-red 'Cloudant in' node (take the dynamic value from input msg) ?

[oprince]

The following minor modification for release 0.2.17, enables dynamic database name:

var dbName = msg.payload.dbName || node.database; var db = cloudant.use(dbName);

msg.payload.dbName should be set to dynamic value at the node preceding to cloudant in node. Would it be possible to implement it for next release ?

[knolleary]

This is a good feature to support given the changes to the IoT Platform.

Note that by convention, if a configuration property has been set in the node, it should not be possible to override it by a message property. If you want to set it via a message property, the value should be left blank in the node's configuration. That means the logic to handle it should be slightly different to what @oprince proposes, but the basic principle is good.

[oprince]

The IoT events DB name is built of fixed value prefix + dynamic suffix representing the current time interval. It would be very helpful if the fixed value prefix is specified in Cloudant node as settings variable, and dynamic suffix is set according to the IoT interval settings. I have implemented that at the following flow which is looping over Events DBs, from newest to oldest, searching for specific events. The IoT service I use is set to 1 month interval. It might be set to 1 week, or 1 day in IoT service settings: [ { "id": "1c31a192.51601e", "type": "http in", "z": "2f5aef1.216631", "name": "", "url": "/iotEvents", "method": "get", "swaggerDoc": "3a12ffa0.faf4", "x": 131.0994415283203, "y": 341, "wires": [ [ "ae2b0115.5e5a" ] ] }, { "id": "ae2b0115.5e5a", "type": "function", "z": "2f5aef1.216631", "name": "prepare cloudant query", "func": "var query = \"deviceId: \" + msg.req.query.deviceId;\nmsg.payload = {\n query: query,\n limit: 100\n}\nreturn msg;", "outputs": 1, "noerr": 0, "x": 354.95448303222656, "y": 342.90057373046875, "wires": [ [ "c2e817e9.17e928" ] ] }, { "id": "77f44aa3.9aae34", "type": "cloudant in", "z": "2f5aef1.216631", "name": "search historical data", "cloudant": "1e7238e0.dbf9f7", "database": "iotp_xkte4l_events_2016-09", "service": "_ext_", "search": "_idx_", "design": "iotp", "index": "search", "x": 384.96031188964844, "y": 434.414794921875, "wires": [ [ "b0f7e035.fbe4" ] ] }, { "id": "c2e817e9.17e928", "type": "function", "z": "2f5aef1.216631", "name": "get historical DB name", "func": "var DBPrefix = \"iotp_IoTOrgId_IoTDBName_\";\nvar now = new Date();\nvar month = now.getMonth() + 1;\nvar year = now.getFullYear();\nnode.warn(\"current month: \" + month + \", year: \" + year);\nif (msg.dbName){\n node.warn(\"msg.dbName found: \" + msg.dbName);\n //Preior cloudant query completed with no items in response\n //Try to get info from preior interval DB\n month = msg.dbName.substr(msg.dbName.length - 2);\n year = msg.dbName.substr(msg.dbName.length - 7, 4);\n node.warn(\"preior month: \" + month + \", year: \" + year);\n month = parseInt(month);\n if (month > 1){\n month --;\n month = month.toString();\n month = (\"00\" + month).slice(-2);\n node.warn(\"new month: \" + month);\n }\n else{\n year = parseInt(year);\n year --;\n month = \"12\";\n }\n}else{\n node.warn(\"msg.dbName not found\");\n}\nmsg.dbName = DBPrefix + year + \"-\" + month;\nmsg.payload.dbName = msg.dbName;\nnode.warn(\"using msg.dbName: \" + msg.dbName);\n\nreturn msg;", "outputs": 1, "noerr": 0, "x": 595.1419982910156, "y": 345.2528381347656, "wires": [ [ "77f44aa3.9aae34" ] ] }, { "id": "b0f7e035.fbe4", "type": "function", "z": "2f5aef1.216631", "name": "check if any events", "func": "var messages = [2];\n\nif (msg.payload){\n if (msg.payload.length > 0){\n messages[0] = msg;\n }else{\n messages[1] = msg;\n }\n}else{\n msg.payload = {\n status: \"not found\"\n }\n messages[0] = msg;\n}\n\nreturn messages;", "outputs": "2", "noerr": 0, "x": 718.1423187255859, "y": 432.2528381347656, "wires": [ [ "889a1c40.2a8b6" ], [ "5e6e7468.81394c" ] ] }, { "id": "5e6e7468.81394c", "type": "link out", "z": "2f5aef1.216631", "name": "continue with older interval DB", "links": [ "d43a290d.1bbcf8" ], "x": 859.1420745849609, "y": 519.3068237304688, "wires": [] }, { "id": "d43a290d.1bbcf8", "type": "link in", "z": "2f5aef1.216631", "name": "", "links": [ "5e6e7468.81394c" ], "x": 171.14480590820312, "y": 533.7215576171875, "wires": [ [ "ae2b0115.5e5a" ] ] }, { "id": "889a1c40.2a8b6", "type": "http response", "z": "2f5aef1.216631", "name": "", "x": 978.1335296630859, "y": 426.48577880859375, "wires": [] }, { "id": "3a12ffa0.faf4", "type": "swagger-doc", "z": "2f5aef1.216631", "summary": "", "description": "", "tags": "", "consumes": "", "produces": "", "parameters": [ { "name": "deviceId", "in": "query", "required": false }, { "name": "historyTime", "in": "query", "required": false }, { "name": "itemId", "in": "query", "required": false }, { "name": "property", "in": "query", "required": false }, { "name": "historyType", "in": "query", "required": false } ], "responses": {}, "deprecated": false }, { "id": "1e7238e0.dbf9f7", "type": "cloudant", "z": "2f5aef1.216631", "host": "d47dbd00-6e11-4907-ba9b-8124e31d147a-bluemix.cloudant.com", "name": "IoT events" } ]