Exporter does not find topic

[TheGim]

My Tasmota devices are displayed and the values arrive as well. But I am currently still trying to get the topics of my ESP01. With MQTT-fx I could verify that the ESP really sends something and the topic is present.

The topics look like this: "dht/livingroom/TEMPERATURE". "dht/livingroom/HUMIDITY"

and send data like: '20.0'.

But the topics do not show up. I'm starting to feel like I'm missing something obvious.

[kpetremann]

hello @TheGim,

Usually the sensor is not in the topic message but in the payload (at least for Xiaomi/Aqara devices and SonOFF): topic '<prefix>/<name>', payload '{"temperature":26.24,"humidity":45.37}'

But it looks like you device is not reporting "the usual" way (I don't there is a standard for this). It seems similar to Shelly's message: topic '<prefix>/<name>/sensor/temperature' '20.00'

Do you know what "dht" stands for? it the device friendly name?

I am thinking about a workaround for your sensor, but I'll try to find something more generic, as I thought Shelly's sensors were the only sensors to post message like this.

[kpetremann]

@TheGim, in addition to the response to my questions, could you please also try if the PR #18 solves your issue?

[TheGim]

Hello @kpetremann,

I think I should explain the situation a little better.

The ESP01 or a generic ESP8266 module is a microcontroller. The dht is the name of the seonsor I'm using, a DHT22. The topic and the payload is custom and so is the whole device. the MQTT exchange takes place via the library: "AsyncMqttClient.h".

My code is based around this:

/*
  Rui Santos
  Complete project details at https://RandomNerdTutorials.com/esp8266-nodemcu-mqtt-publish-dht11-dht22-arduino/

  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files.

  The above copyright notice and this permission notice shall be included in all
  copies or substantial portions of the Software.
*/

#include "DHT.h"
#include <ESP8266WiFi.h>
#include <Ticker.h>
#include <AsyncMqttClient.h>

#define WIFI_SSID "REPLACE_WITH_YOUR_SSID"
#define WIFI_PASSWORD "REPLACE_WITH_YOUR_PASSWORD"

// Raspberri Pi Mosquitto MQTT Broker
#define MQTT_HOST IPAddress(192, 168, 1, XXX)
// For a cloud MQTT broker, type the domain name
//#define MQTT_HOST "example.com"
#define MQTT_PORT 1883

// Temperature MQTT Topics
#define MQTT_PUB_TEMP "esp/dht/temperature"
#define MQTT_PUB_HUM "esp/dht/humidity"

// Digital pin connected to the DHT sensor
#define DHTPIN 14  

// Uncomment whatever DHT sensor type you're using
//#define DHTTYPE DHT11   // DHT 11
#define DHTTYPE DHT22   // DHT 22  (AM2302), AM2321
//#define DHTTYPE DHT21   // DHT 21 (AM2301)   

// Initialize DHT sensor
DHT dht(DHTPIN, DHTTYPE);

// Variables to hold sensor readings
float temp;
float hum;

AsyncMqttClient mqttClient;
Ticker mqttReconnectTimer;

WiFiEventHandler wifiConnectHandler;
WiFiEventHandler wifiDisconnectHandler;
Ticker wifiReconnectTimer;

unsigned long previousMillis = 0;   // Stores last time temperature was published
const long interval = 10000;        // Interval at which to publish sensor readings

void connectToWifi() {
  Serial.println("Connecting to Wi-Fi...");
  WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
}

void onWifiConnect(const WiFiEventStationModeGotIP& event) {
  Serial.println("Connected to Wi-Fi.");
  connectToMqtt();
}

void onWifiDisconnect(const WiFiEventStationModeDisconnected& event) {
  Serial.println("Disconnected from Wi-Fi.");
  mqttReconnectTimer.detach(); // ensure we don't reconnect to MQTT while reconnecting to Wi-Fi
  wifiReconnectTimer.once(2, connectToWifi);
}

void connectToMqtt() {
  Serial.println("Connecting to MQTT...");
  mqttClient.connect();
}

void onMqttConnect(bool sessionPresent) {
  Serial.println("Connected to MQTT.");
  Serial.print("Session present: ");
  Serial.println(sessionPresent);
}

void onMqttDisconnect(AsyncMqttClientDisconnectReason reason) {
  Serial.println("Disconnected from MQTT.");

  if (WiFi.isConnected()) {
    mqttReconnectTimer.once(2, connectToMqtt);
  }
}

/*void onMqttSubscribe(uint16_t packetId, uint8_t qos) {
  Serial.println("Subscribe acknowledged.");
  Serial.print("  packetId: ");
  Serial.println(packetId);
  Serial.print("  qos: ");
  Serial.println(qos);
}

void onMqttUnsubscribe(uint16_t packetId) {
  Serial.println("Unsubscribe acknowledged.");
  Serial.print("  packetId: ");
  Serial.println(packetId);
}*/

void onMqttPublish(uint16_t packetId) {
  Serial.print("Publish acknowledged.");
  Serial.print("  packetId: ");
  Serial.println(packetId);
}

void setup() {
  Serial.begin(115200);
  Serial.println();

  dht.begin();

  wifiConnectHandler = WiFi.onStationModeGotIP(onWifiConnect);
  wifiDisconnectHandler = WiFi.onStationModeDisconnected(onWifiDisconnect);

  mqttClient.onConnect(onMqttConnect);
  mqttClient.onDisconnect(onMqttDisconnect);
  //mqttClient.onSubscribe(onMqttSubscribe);
  //mqttClient.onUnsubscribe(onMqttUnsubscribe);
  mqttClient.onPublish(onMqttPublish);
  mqttClient.setServer(MQTT_HOST, MQTT_PORT);
  // If your broker requires authentication (username and password), set them below
  //mqttClient.setCredentials("REPlACE_WITH_YOUR_USER", "REPLACE_WITH_YOUR_PASSWORD");

  connectToWifi();
}

void loop() {
  unsigned long currentMillis = millis();
  // Every X number of seconds (interval = 10 seconds) 
  // it publishes a new MQTT message
  if (currentMillis - previousMillis >= interval) {
    // Save the last time a new reading was published
    previousMillis = currentMillis;
    // New DHT sensor readings
    hum = dht.readHumidity();
    // Read temperature as Celsius (the default)
    temp = dht.readTemperature();
    // Read temperature as Fahrenheit (isFahrenheit = true)
    //temp = dht.readTemperature(true);

    // Publish an MQTT message on topic esp/dht/temperature
    uint16_t packetIdPub1 = mqttClient.publish(MQTT_PUB_TEMP, 1, true, String(temp).c_str());                            
    Serial.printf("Publishing on topic %s at QoS 1, packetId: %i ", MQTT_PUB_TEMP, packetIdPub1);
    Serial.printf("Message: %.2f \n", temp);

    // Publish an MQTT message on topic esp/dht/humidity
    uint16_t packetIdPub2 = mqttClient.publish(MQTT_PUB_HUM, 1, true, String(hum).c_str());                            
    Serial.printf("Publishing on topic %s at QoS 1, packetId %i: ", MQTT_PUB_HUM, packetIdPub2);
    Serial.printf("Message: %.2f \n", hum);
  }

[TheGim]

I will try #18

[TheGim]

I tried #18. Unfortunately without success.

[kpetremann]

ok, could you please provide me the exact message posted by your sensor to MQTT please?

[TheGim]

Topic	Message
dht/livingroom/temperature	"20.1"

[kpetremann]

weird, this is exactly what I used for the test: https://github.com/kpetremann/mqtt-exporter/pull/18/commits/d49375dca583be5bad9bced7e9a1f27f380ca01e#diff-a92777fcddfc4692e4b729e914f083aee15a41a915bee2889661822f81435087

PS: I'll update the PR so it mention the fact it is to support custom implementation and not the official Tasmota MQTT messages (https://tasmota.github.io/docs/MQTT/), which should be already supported.

[kpetremann]

do you have any relevant logs in the mqtt-exporter logs (with LOG_LEVEL set to "debug" value)?

I'll try your code

[TheGim]

I tried it! and there are values that overlap. But I don't get it, why it should be an unexpected format

DEBUG:mqtt-exporter:failed to parse as JSON: "b'nan'"
DEBUG:mqtt-exporter:failed to parse as JSON: "b'ON'"
DEBUG:mqtt-exporter:unexpected payload format: "21.0"
DEBUG:mqtt-exporter:unexpected payload format: "48.2"
DEBUG:mqtt-exporter:failed to parse as JSON: "b'nan'"
DEBUG:mqtt-exporter:failed to parse as JSON: "b'nan'"
DEBUG:mqtt-exporter:unexpected payload format: "22.1"
DEBUG:mqtt-exporter:unexpected payload format: "52.3"

[kpetremann]

I should rewrite the log message. Now this is more failed to parse: topic is '%s' payload '%s'

Let me update the PR with this change so you can test with more info

Regarding the other message (failed to parse as JSON) it looks like the lib is not able to parse bytes, while it works in the tests, weird.

[kpetremann]

I have updated the PR, please test with it and provide the output.

[TheGim]

nothing has changed in terms of temperature and humidity values. The rest of the data comes from Octoprint, which can be ignored.

INFO:mqtt-exporter:creating prometheus metric: mqtt_progress
DEBUG:mqtt-exporter:new value for mqtt_progress: 100
DEBUG:mqtt-exporter:new value for mqtt__timestamp: 1636365275
DEBUG:mqtt-exporter:failed to parse as JSON: "b'Online'"
DEBUG:mqtt-exporter:failed to parse as JSON: "b'Online'"
DEBUG:mqtt-exporter:failed to parse as JSON: "b'nan'"
DEBUG:mqtt-exporter:failed to parse as JSON: "b'ON'"
DEBUG:mqtt-exporter:unexpected payload format: "21.0"
DEBUG:mqtt-exporter:unexpected payload format: "48.2"
DEBUG:mqtt-exporter:unexpected payload format: "22.2"
DEBUG:mqtt-exporter:unexpected payload format: "52.8"
DEBUG:mqtt-exporter:unexpected payload format: "22.1"
DEBUG:mqtt-exporter:unexpected payload format: "52.3"
DEBUG:mqtt-exporter:unexpected payload format: "22.2"
DEBUG:mqtt-exporter:unexpected payload format: "52.9"

[kpetremann]

Looks like your are not using the code from the PR. As an alternative you can just switch to the tasmota branch.

[TheGim]

You are right!

But there seems to be an Error

Traceback (most recent call last):
  File "exporter.py", line 5, in <module>
    main()
  File "/home/pi/mqtt-exporter/mqtt_exporter/main.py", line 201, in main
    client.loop_forever()
  File "/home/pi/.local/lib/python3.8/site-packages/paho/mqtt/client.py", line 1756, in loop_forever
    rc = self._loop(timeout)
  File "/home/pi/.local/lib/python3.8/site-packages/paho/mqtt/client.py", line 1164, in _loop
    rc = self.loop_read()
  File "/home/pi/.local/lib/python3.8/site-packages/paho/mqtt/client.py", line 1556, in loop_read
    rc = self._packet_read()
  File "/home/pi/.local/lib/python3.8/site-packages/paho/mqtt/client.py", line 2439, in _packet_read
    rc = self._packet_handle()
  File "/home/pi/.local/lib/python3.8/site-packages/paho/mqtt/client.py", line 3033, in _packet_handle
    return self._handle_publish()
  File "/home/pi/.local/lib/python3.8/site-packages/paho/mqtt/client.py", line 3327, in _handle_publish
    self._handle_on_message(message)
  File "/home/pi/.local/lib/python3.8/site-packages/paho/mqtt/client.py", line 3570, in _handle_on_message
    on_message(self, self._userdata, message)
  File "/home/pi/mqtt-exporter/mqtt_exporter/main.py", line 160, in expose_metrics
    topic, payload = _parse_message(msg.topic, msg.payload)
  File "/home/pi/mqtt-exporter/mqtt_exporter/main.py", line 134, in _parse_message
    if not isinstance(payload, dict):
UnboundLocalError: local variable 'payload' referenced before assignment

[TheGim]

It was my mistake, I switched to the last commit and it works. The advanced debug function was not necessary, sorry. But I'm sure it will be useful in the future.

Thank you for your help!

[kpetremann]

ok cool perfect :)

On a side note, as you are developing your own tool, I would suggest to pack all the metrics of the sensor into a single message, using a JSON structure.

have fun with your project :)

[TheGim]

Thank you for the suggestion. I just implemented it. This will reduce the networkrequests from 2 to 1. :)

[kpetremann]

@TheGim FYI the PR has been merged