thingsboard / thingsboard-gateway

Open-source IoT Gateway - integrates devices connected to legacy and third-party systems with ThingsBoard IoT Platform using Modbus, CAN bus, BACnet, BLE, OPC-UA, MQTT, ODBC and REST protocols
https://thingsboard.io/docs/iot-gateway/what-is-iot-gateway/
Apache License 2.0
1.66k stars 801 forks source link

[HELP]mqtt data convert too long #1433

Open Leo-Hassan opened 3 weeks ago

Leo-Hassan commented 3 weeks ago

Describe the issue Create description about your issue, and your actions to solve it.

collect the data once per second form mqtt broker 172.17.0.5 need send to 172.17.0.2(thingsbroard) rigth now

When the data publishing frequency of MQTT is more than 10 seconds, the data is displayed without loss or in a normal state. When the publishing frequency of the data is once per second, the data is displayed with a 10-second delay. The log shows that it is waiting to connect to the broker... I need to collect the data once per second and display it to ThingsBoard. At least it should not be displayed once every 10 seconds.

tb_gateway.json { "thingsboard": { "host": "172.17.0.2", "port": 1883, "remoteShell": false, "remoteConfiguration": true, "statistics": { "enable": true, "statsSendPeriodInSeconds": 3600, "configuration": null, "commands": [] }, "deviceFiltering": { "enable": false, "filterFile": "list.json" }, "maxPayloadSizeBytes": 1024, "minPackSendDelayMS": 200, "minPackSizeToSend": 500, "checkConnectorsConfigurationInSeconds": 60, "handleDeviceRenaming": true, "security": { "accessToken": "no care" }, "qos": 1, "checkingDeviceActivity": { "checkDeviceInactivity": false, "inactivityTimeoutSeconds": 200, "inactivityCheckPeriodSeconds": 500 }, "ts": 1718966089915, "rateLimits": "DEFAULT_RATE_LIMIT", "dpRateLimits": "DEFAULT_RATE_LIMIT" }, "storage": { "type": "memory", "read_records_count": 100, "max_records_count": 100000, "data_folder_path": "./data/", "max_file_count": 10, "max_read_records_count": 10, "max_records_per_file": 10000, "data_file_path": "./data/data.db", "messages_ttl_check_in_hours": 1, "messages_ttl_in_days": 7, "ts": 1718966089915 }, "grpc": { "enabled": true, "serverPort": 9595, "keepaliveTimeMs": 10000, "keepaliveTimeoutMs": 5000, "keepalivePermitWithoutCalls": true, "maxPingsWithoutData": 0, "minTimeBetweenPingsMs": 10000, "minPingIntervalWithoutDataMs": 5000, "keepAliveTimeMs": 10000, "keepAliveTimeoutMs": 5000, "ts": 1718961607048 }, "connectors": [ { "type": "mqtt", "name": "mqtt1", "configuration": "mqtt1.json" } ] }

mqtt.json

{ "broker": { "name": "Default Local Broker", "host": "172.17.0.5", "port": 1883, "version": 4, "clientId": "tb_gw_1sede", "maxNumberOfWorkers": 100, "maxMessageNumberPerWorker": 10, "security": { "type": "anonymous" } }, "dataMapping": [ { "topicFilter": "sensor/data", "subscriptionQos": 1, "converter": { "type": "json", "deviceInfo": { "deviceNameExpressionSource": "message", "deviceNameExpression": "${serialNumber}", "deviceProfileExpressionSource": "message", "deviceProfileExpression": "${sensorType}" }, "sendDataOnlyOnChange": false, "timeout": 60000, "attributes": [ { "type": "string", "key": "model", "value": "${sensorModel}" }, { "type": "string", "key": "${sensorModel}", "value": "on" } ], "timeseries": [ { "type": "string", "key": "temperature", "value": "${temp}" }, { "type": "double", "key": "humidity", "value": "${hum}" }, { "type": "string", "key": "combine", "value": "${hum}:${temp}" } ] } }, { "topicFilter": "sensor/+/data", "subscriptionQos": 1, "converter": { "type": "json", "deviceInfo": { "deviceNameExpressionSource": "topic", "deviceNameExpression": "(?<=sensor/)(.?)(?=/data)", "deviceProfileExpressionSource": "constant", "deviceProfileExpression": "Thermometer" }, "sendDataOnlyOnChange": false, "timeout": 60000, "attributes": [ { "type": "string", "key": "model", "value": "${sensorModel}" } ], "timeseries": [ { "type": "double", "key": "temperature", "value": "${temp}" }, { "type": "string", "key": "humidity", "value": "${hum}" } ] } }, { "topicFilter": "sensor/raw_data", "subscriptionQos": 1, "converter": { "type": "bytes", "deviceInfo": { "deviceNameExpressionSource": "message", "deviceNameExpression": "[0:4]", "deviceProfileExpressionSource": "constant", "deviceProfileExpression": "default" }, "sendDataOnlyOnChange": false, "timeout": 60000, "attributes": [ { "type": "raw", "key": "rawData", "value": "[:]" } ], "timeseries": [ { "type": "raw", "key": "temp", "value": "[4:]" } ] } }, { "topicFilter": "custom/sensors/+", "subscriptionQos": 1, "converter": { "type": "custom", "extension": "CustomMqttUplinkConverter", "cached": true, "extensionConfig": { "temperature": 2, "humidity": 2, "batteryLevel": 1 } } } ], "requestsMapping": { "connectRequests": [ { "topicFilter": "sensor/connect", "deviceInfo": { "deviceNameExpressionSource": "message", "deviceNameExpression": "${serialNumber}", "deviceProfileExpressionSource": "constant", "deviceProfileExpression": "Thermometer" } }, { "topicFilter": "sensor/+/connect", "deviceInfo": { "deviceNameExpressionSource": "topic", "deviceNameExpression": "(?<=sensor/)(.?)(?=/connect)", "deviceProfileExpressionSource": "constant", "deviceProfileExpression": "Thermometer" } } ], "disconnectRequests": [ { "topicFilter": "sensor/disconnect", "deviceInfo": { "deviceNameExpressionSource": "message", "deviceNameExpression": "${serialNumber}" } }, { "topicFilter": "sensor/+/disconnect", "deviceInfo": { "deviceNameExpressionSource": "topic", "deviceNameExpression": "(?<=sensor/)(.?)(?=/connect)" } } ], "attributeRequests": [ { "retain": false, "topicFilter": "v1/devices/me/attributes/request", "deviceInfo": { "deviceNameExpressionSource": "message", "deviceNameExpression": "${serialNumber}" }, "attributeNameExpressionSource": "message", "attributeNameExpression": "${versionAttribute}, ${pduAttribute}", "topicExpression": "devices/${deviceName}/attrs", "valueExpression": "${attributeKey}: ${attributeValue}" } ], "attributeUpdates": [ { "retain": true, "deviceNameFilter": ".", "attributeFilter": "firmwareVersion", "topicExpression": "sensor/${deviceName}/${attributeKey}", "valueExpression": "{\"${attributeKey}\":\"${attributeValue}\"}" } ], "serverSideRpc": [ { "type": "twoWay", "deviceNameFilter": ".", "methodFilter": "echo", "requestTopicExpression": "sensor/${deviceName}/request/${methodName}/${requestId}", "responseTopicExpression": "sensor/${deviceName}/response/${methodName}/${requestId}", "responseTopicQoS": 1, "responseTimeout": 10000, "valueExpression": "${params}" }, { "type": "oneWay", "deviceNameFilter": ".", "methodFilter": "no-reply", "requestTopicExpression": "sensor/${deviceName}/request/${methodName}/${requestId}", "valueExpression": "${params}" } ] }, "logLevel": "INFO", "name": "mqtt1", "enableRemoteLogging": false, "id": "eb133ff9-91fb-404e-b08f-6aa337eb0a6a" }

Configuration (Attach your configuration file) Notate: Remove Access token from file if you want to attach a tb_gateway.yaml

Connector name (If you need help with some connector/converter): [e.g. MQTT Connector]

MQTT Connector

Versions (please complete the following information):

imbeacon commented 2 weeks ago

Hi @Leo-Hassan,

Could you provide an example of payload that you are trying to collect? We will try to reproduce the issue.