modi12jin
commented
1 year ago sorry for the stupid question
host_ip = '192.168.0.108'
port = 8000
servver = await websockets.serve(data_handler, host_ip, port)Closed modi12jin closed 1 year ago
modi12jin
commented
1 year ago sorry for the stupid question
host_ip = '192.168.0.108'
port = 8000
servver = await websockets.serve(data_handler, host_ip, port)@DaveBben Is this output normal?
out of range
out of range
out of range
out of range
out of range
Max frame size for video is 19225 with a total a total of 297 frames
Max frame size for audio is 2940 with a total a total of 292 frames
Buffers filled
websockets.exceptions.ConnectionClosedError: received 1009 (message too big); then sent 1009 (message too big)esp32s3_client_code.ino
#include "JPEGDEC.h"
#include <WiFi.h>
#include "LovyanGFX_Driver.h"
#include "i2s_init.h"
#include "driver/gpio.h"
#include <WebSocketsClient.h>
#include <time.h>
#include <sys/time.h>
#include "esp_sntp.h"
#define I2S_DOUT 37
#define I2S_BCLK 36
#define I2S_LRC 35
#define HEADER_MESSAGE_LENGTH 2 //标题信息长度
const uint16_t VIDEO_FRAME_BUFFER_SIZE = 20000; //视频帧缓冲区大小
const uint16_t AUDIO_FRAME_BUFFER_SIZE = 4000; //音频帧缓冲区大小
const char *ssid = "FAST_BA74";// SSID
const char *password = "12345678";// Password
const char *websockets_server_host = "192.168.0.109";// server adress
const uint16_t port = 8000;
WebSocketsClient websocketClient;
JPEGDEC jpeg;
LGFX tft;
struct Frame {
uint8_t video_data[VIDEO_FRAME_BUFFER_SIZE];
uint8_t audio_data[AUDIO_FRAME_BUFFER_SIZE];
uint16_t video_data_size;
uint16_t audio_data_size;
};
/**
* @brief
* 使用循环缓冲区来存储和读取帧
*/
struct Circular_Buffer {
uint8_t start_pointer;
uint8_t end_pointer;
struct Frame *items;
};
const uint8_t CIRCULAR_BUFFER_SIZE = 4; //圆形缓冲区大小
int JPEGDraw(JPEGDRAW *pDraw);
void render_image(void *parameter);
void socket_loop(void *parameter);
struct Circular_Buffer *data_buffer;
uint8_t video_frame_buffer[VIDEO_FRAME_BUFFER_SIZE] = { 0 };
uint8_t audio_frame_buffer[AUDIO_FRAME_BUFFER_SIZE] = { 0 };
unsigned long lastUpdate = 0;
volatile int fps = 0;
volatile int updates = 0;
volatile uint16_t buttonPressed = 0;
volatile unsigned long lastDebounceTime = 0;
//缓冲区为空
uint8_t isBufferEmpty(struct Circular_Buffer *circular_buff) {
if (circular_buff->end_pointer == circular_buff->start_pointer) {
return 1;
} else {
return 0;
}
}
//缓冲区已满
uint8_t isBufferFull(struct Circular_Buffer *circular_buff) {
if (((circular_buff->end_pointer + 1) % CIRCULAR_BUFFER_SIZE) == circular_buff->start_pointer) {
return 1;
} else {
return 0;
}
}
//读取数据帧
void read_data_frame(uint8_t *video_buffer, uint16_t *video_size, uint8_t *audio_buffer, uint16_t *audio_size, Circular_Buffer *circular_buffer) {
if (!isBufferEmpty(circular_buffer)) {
struct Frame *single_frame = &circular_buffer->items[circular_buffer->start_pointer];
memcpy(video_buffer, single_frame->video_data, single_frame->video_data_size);
memcpy(audio_buffer, single_frame->audio_data, single_frame->audio_data_size);
*video_size = single_frame->video_data_size;
*audio_size = single_frame->audio_data_size;
circular_buffer->start_pointer = (circular_buffer->start_pointer + 1) % CIRCULAR_BUFFER_SIZE;
}
}
//重置缓冲区
void reset_buffer(struct Circular_Buffer *circular_buff) {
circular_buff->end_pointer = 1;
circular_buff->start_pointer = 1;
}
//由于我们没有使用按钮，因此不需要
static void IRAM_ATTR gpio_isr(void *arg) {
if ((millis() - lastDebounceTime) > 3000) {
lastDebounceTime = millis();
buttonPressed = 1;
}
}
//没有使用，因为没有按钮
void request_video() {
websocketClient.sendTXT("play");
}
//websocket事件
void websocket_event(WStype_t type, uint8_t *payload, size_t length) {
switch (type) {
case WStype_DISCONNECTED:
Serial.printf("[WSc] Disconnected!\n");
reset_buffer(data_buffer);
// vTaskDelay(100 / portTICK_RATE_MS);
i2s_zero_dma_buffer(I2S_NUM_0);
break;
case WStype_CONNECTED:
Serial.printf("[WSc] Connected to url: %s\n", payload);
websocketClient.sendTXT("play");
break;
case WStype_TEXT:
Serial.printf("[WSc] get text: %s\n", payload);
if (strcmp((char *)payload, "done") == 0) {
// This delay is a lazy hack
// basically need to wait until
// all the i2s transfers are actually
// done
// vTaskDelay(100 / portTICK_RATE_MS);
i2s_zero_dma_buffer(I2S_NUM_0);
}
break;
case WStype_BIN:
if (!isBufferFull(data_buffer)) {
if (length > 0) {
//The data will come in packaged. We want to deconstruct and put
// into audio/video frame
struct Frame *single_frame = &data_buffer->items[data_buffer->end_pointer];
// // first two bytes represent video length
uint16_t video_length = ((uint16_t)payload[0] << 8) | payload[1];
single_frame->video_data_size = video_length;
// Serial.printf("Video length is %d\n", video_length);
memcpy(single_frame->video_data, payload + HEADER_MESSAGE_LENGTH, video_length);
uint16_t audio_length = length - video_length - HEADER_MESSAGE_LENGTH;
single_frame->audio_data_size = audio_length;
// Serial.printf("Audio length is %d\n", audio_length);
memcpy(single_frame->audio_data, payload + HEADER_MESSAGE_LENGTH + video_length, audio_length);
data_buffer->end_pointer = (data_buffer->end_pointer + 1) % CIRCULAR_BUFFER_SIZE;
}
} else {
Serial.println("Buffer is full!");
}
break;
}
}
int JPEGDraw(JPEGDRAW *pDraw) {
if (tft.getStartCount() == 0) {
tft.endWrite();
}
tft.pushImageDMA(pDraw->x, pDraw->y, pDraw->iWidth, pDraw->iHeight, (lgfx::swap565_t *)pDraw->pPixels);
return 1;
}
//处理视频
void handle_video(void *pvParameter) {
for (;;) { // infinite loop
if (!isBufferEmpty(data_buffer)) {
uint16_t video_length, audio_length;
read_data_frame(video_frame_buffer, &video_length, audio_frame_buffer, &audio_length, data_buffer);
int i;
tft.startWrite();
if (jpeg.openRAM(video_frame_buffer, video_length, JPEGDraw)) {
jpeg.setPixelType(RGB565_BIG_ENDIAN);
// lTime = micros();
if (jpeg.decode(0, 0, 0)) {
tft.endWrite();
updates += 1;
size_t written = 0;
i2s_write(I2S_NUM_0, audio_frame_buffer, audio_length, &written, portMAX_DELAY);
} else {
Serial.println("Decode error");
}
jpeg.close();
}
}
unsigned long time = millis();
if (time - lastUpdate >= 1000) {
float overtime = float(time - lastUpdate) / 1000.0;
fps = floor((float)updates / overtime);
lastUpdate = time;
updates = 0;
Serial.print("FPS: ");
Serial.println(fps);
}
}
}
void setup() {
data_buffer = (struct Circular_Buffer *)malloc(sizeof(struct Circular_Buffer));
data_buffer->items = (struct Frame *)malloc(sizeof *data_buffer->items * CIRCULAR_BUFFER_SIZE);
data_buffer->end_pointer = 1;
data_buffer->start_pointer = 1;
Serial.begin(115200);
if (data_buffer == NULL or data_buffer->items == NULL) {
Serial.println("Failed to allocate memory on heap");
return;
}
WiFi.begin(ssid, password);
// Wait some time to connect to wifi
for (int i = 0; i < 10 && WiFi.status() != WL_CONNECTED; i++) {
Serial.print(".");
delay(1000);
}
// Check if connected to wifi
if (WiFi.status() != WL_CONNECTED) {
Serial.println("No Wifi!");
return;
}
tft.begin();
tft.initDMA();
tft.fillScreen(TFT_BLACK);
websocketClient.begin(websockets_server_host, port, "/");
websocketClient.onEvent(websocket_event);
lastUpdate = millis();
esp_err_t ret_val = i2s_init(I2S_NUM_0, 44100, -1, I2S_BCLK, I2S_LRC, I2S_DOUT, -1);
if (ret_val != ESP_OK) {
Serial.printf("i2s_init failed: %d\n", ret_val);
// tft.println("i2s_init failed");
return;
}
i2s_zero_dma_buffer(I2S_NUM_0);
size_t written = 0;
i2s_start(I2S_NUM_0);
// No button, so not using the following
// install gpio isr service
// gpio_install_isr_service(0);
// hook isr handler for specific gpio pin
// gpio_config_t io_conf = {};
// io_conf.intr_type = GPIO_INTR_POSEDGE;
// io_conf.mode = GPIO_MODE_INPUT;
// io_conf.pull_down_en = GPIO_PULLDOWN_DISABLE;
// io_conf.pull_up_en = GPIO_PULLUP_DISABLE;
// io_conf.pin_bit_mask = (1ULL << GPIO_NUM_17);
// gpio_config(&io_conf);
// gpio_isr_handler_add(GPIO_NUM_17, gpio_isr, (void *)GPIO_NUM_17);
xTaskCreatePinnedToCore(
socket_loop, /* Function to implement the task */
"handle_loop", /* Name of the task */
4096, /* Stack size in words */
NULL, /* Task input parameter */
1, /* Priority of the task */
NULL, /* Task handle. */
0); /* Core where the task should run */
xTaskCreatePinnedToCore(
handle_video, /* Function to implement the task */
"handle_video", /* Name of the task */
4096, /* Stack size in words */
NULL, /* Task input parameter */
configMAX_PRIORITIES - 1, /* Priority of the task */
NULL, /* Task handle. */
1); /* Core where the task should run */
}
void loop() {
}
void socket_loop(void *parameter) {
for (;;) { // infinite loop
while (!isBufferFull(data_buffer)) {
if (buttonPressed) {
buttonPressed = 0;
websocketClient.sendTXT("play");
}
websocketClient.loop();
vTaskDelay(5 / portTICK_RATE_MS);
}
vTaskDelay(10 / portTICK_RATE_MS);
}
}LovyanGFX_Driver.h
#define LGFX_USE_V1
#include <LovyanGFX.hpp>
class LGFX : public lgfx::LGFX_Device {
lgfx::Panel_ST7796 _panel_instance;
lgfx::Bus_Parallel8 _bus_instance;
lgfx::Light_PWM _light_instance;
public:
LGFX(void) {
{
auto cfg = _bus_instance.config();
cfg.port = 0;
cfg.freq_write = 20000000;
cfg.pin_wr = 47; // WR を接続しているピン番号
cfg.pin_rd = -1; // RD を接続しているピン番号
cfg.pin_rs = 0; // RS(D/C)を接続しているピン番号
cfg.pin_d0 = 9; // D0を接続しているピン番号
cfg.pin_d1 = 46; // D1を接続しているピン番号
cfg.pin_d2 = 3; // D2を接続しているピン番号
cfg.pin_d3 = 8; // D3を接続しているピン番号
cfg.pin_d4 = 18; // D4を接続しているピン番号
cfg.pin_d5 = 17; // D5を接続しているピン番号
cfg.pin_d6 = 16; // D6を接続しているピン番号
cfg.pin_d7 = 15; // D7を接続しているピン番号
_bus_instance.config(cfg); // 設定値をバスに反映します。
_panel_instance.setBus(&_bus_instance); // バスをパネルにセットします。
}
{ // 表示パネル制御の設定を行います。
auto cfg = _panel_instance.config(); // 表示パネル設定用の構造体を取得します。
cfg.pin_cs = -1; // CSが接続されているピン番号 (-1 = disable)
cfg.pin_rst = 4; // RSTが接続されているピン番号 (-1 = disable)
cfg.pin_busy = -1; // BUSYが接続されているピン番号 (-1 = disable)
// ※ 以下の設定値はパネル毎に一般的な初期値が設定さ BUSYが接続されているピン番号 (-1 = disable)れていますので、不明な項目はコメントアウトして試してみてください。
cfg.memory_width = 320; // ドライバICがサポートしている最大の幅
cfg.memory_height = 480; // ドライバICがサポートしている最大の高さ
cfg.panel_width = 320; // 実際に表示可能な幅
cfg.panel_height = 480; // 実際に表示可能な高さ
cfg.offset_x = 0; // パネルのX方向オフセット量
cfg.offset_y = 0; // パネルのY方向オフセット量
cfg.offset_rotation = 0; //值在旋转方向的偏移0~7（4~7是倒置的）
cfg.dummy_read_pixel = 8; // 在读取像素之前读取的虚拟位数
cfg.dummy_read_bits = 1; // 读取像素以外的数据之前的虚拟读取位数
cfg.readable = false; // 如果可以读取数据，则设置为 true
cfg.invert = true; // 如果面板的明暗反转，则设置为 true
cfg.rgb_order = false; // 如果面板的红色和蓝色被交换，则设置为 true
cfg.dlen_16bit = false; // 对于以 16 位单位发送数据长度的面板，设置为 true
cfg.bus_shared = false; // 如果总线与 SD 卡共享，则设置为 true（使用 drawJpgFile 等执行总线控制）
_panel_instance.config(cfg);
}
{ // バックライト制御の設定を行います。（必要なければ削除）
auto cfg = _light_instance.config(); // バックライト設定用の構造体を取得します。
cfg.pin_bl = 45; // バックライトが接続されているピン番号
cfg.invert = false; // バックライトの輝度を反転させる場合 true
cfg.freq = 44100; // バックライトのPWM周波数
cfg.pwm_channel = 1; // 使用するPWMのチャンネル番号
_light_instance.config(cfg);
_panel_instance.setLight(&_light_instance); // バックライトをパネルにセットします。
}
setPanel(&_panel_instance); // 使用するパネルをセットします。
}
};i2s_init.h
#include "driver/i2s.h"
static i2s_port_t _i2s_num;
static esp_err_t i2s_init(i2s_port_t i2s_num, uint32_t sample_rate,
int mck_io_num, /*!< MCK in out pin. Note that ESP32 supports setting MCK on GPIO0/GPIO1/GPIO3 only*/
int bck_io_num, /*!< BCK in out pin*/
int ws_io_num, /*!< WS in out pin*/
int data_out_num, /*!< DATA out pin*/
int data_in_num /*!< DATA in pin*/
)
{
_i2s_num = i2s_num;
esp_err_t ret_val = ESP_OK;
i2s_config_t i2s_config;
i2s_config.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_TX);
i2s_config.sample_rate = sample_rate;
i2s_config.bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT;
i2s_config.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT;
i2s_config.communication_format = i2s_comm_format_t(I2S_COMM_FORMAT_I2S_MSB);
i2s_config.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1;
i2s_config.dma_buf_count = 64;
i2s_config.dma_buf_len = 50;
i2s_config.use_apll = false;
i2s_config.tx_desc_auto_clear = true;
i2s_config.fixed_mclk = 0;
i2s_config.mclk_multiple = I2S_MCLK_MULTIPLE_DEFAULT;
i2s_config.bits_per_chan = I2S_BITS_PER_CHAN_16BIT;
i2s_pin_config_t pin_config;
pin_config.mck_io_num = mck_io_num;
pin_config.bck_io_num = bck_io_num;
pin_config.ws_io_num = ws_io_num;
pin_config.data_out_num = data_out_num;
pin_config.data_in_num = data_in_num;
ret_val |= i2s_driver_install(i2s_num, &i2s_config, 0, NULL);
ret_val |= i2s_set_pin(i2s_num, &pin_config);
return ret_val;
}@DaveBben Possibly related to this issue
modi12jin
commented
1 year ago @DaveBben It worked, just very stuck.
When I built the ESP remote control, I thought about using the web socket to receive images and transmit control commands. But then gave up: [ESP32 Remote Control ESP32-Bilibili] https://b23.tv/bY8JTAM
As a result, I established two HTTP connections to do two things separately
If you have time, you can try to refer to this: https://github.com/moononournation/ESP32VideoRemote/
This may be the direction of follow-up efforts
I am new to this, how can I solve this problem?