search

espressif / esp-adf

Espressif Audio Development Framework
Other
1.52k stars 669 forks source link

Issue with downmix_pipeline example, base_stream alone : slow+skips, base_stream+newcome_stream: OK (AUD-4856) #1069

Closed 0x0fe closed 11 months ago

0x0fe commented 1 year ago

base file 44.1KHz 2ch tone file 44.1KHz 2ch I2S codec 44.1KHz 2ch (can accept other sample rates too) ADF version 2.6, IDF version 5.0.2 The SD card : SDIO 1bit mode. Everything works fine when testing sd_mmc example

so here is the modified downmix_pipeline example, the mp3 files to use should be both 44.1KHz.

What i modified : -removing all resamplers -adjusting all samplerates to 44.1KHz -adjusting the downmixer configuration to 2*2 channels

The issue is : -when base_stream is playing alone, it is played too slow and skips. -when base_stream and newcome_stream are playing together, everything is fine, playback speed is normal, no skipping.

It can be reproduced anytime the key is pressed, consistently. i cannot find anything wrong in the configuration so i assume it is a bug. The issue occurs exactly the same way when downsamplers are enabled, so they are not relevant to the issue. 

#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "freertos/event_groups.h"
#include "esp_log.h"
#include "audio_element.h"
#include "audio_pipeline.h"
#include "audio_event_iface.h"
#include "i2s_stream.h"
#include "mp3_decoder.h"
#include "fatfs_stream.h"
#include "downmix.h"
#include "filter_resample.h"
#include "raw_stream.h"
#include "board.h"
#include "periph_sdcard.h"
#include "periph_button.h"
#include "input_key_service.h"
#include "periph_adc_button.h"

static const char *TAG = "DOWNMIX_PIPELINE_EXAMPLE";

#define INDEX_BASE_STREAM 0
#define INDEX_NEWCOME_STREAM 1
#define SAMPLERATE 44100
#define NUM_INPUT_CHANNEL 2
#define TRANSMITTIME 500
#define MUSIC_GAIN_DB 0
#define PLAY_STATUS ESP_DOWNMIX_OUTPUT_TYPE_TWO_CHANNEL //ESP_DOWNMIX_OUTPUT_TYPE_ONE_CHANNEL
#define NUMBER_SOURCE_FILE 2

void app_main(void)
{
    audio_element_handle_t base_fatfs_reader_el = NULL;
    audio_element_handle_t base_mp3_decoder_el = NULL;
    //audio_element_handle_t base_rsp_filter_el = NULL;
    audio_element_handle_t base_raw_write_el = NULL;

    audio_element_handle_t newcome_fatfs_reader_el = NULL;
    audio_element_handle_t newcome_mp3_decoder_el = NULL;
    //audio_element_handle_t newcome_rsp_filter_el = NULL;
    audio_element_handle_t newcome_raw_write_el = NULL;

    esp_log_level_set("*", ESP_LOG_WARN);
    esp_log_level_set(TAG, ESP_LOG_INFO);

    audio_board_handle_t board_handle = audio_board_init();
    audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_DECODE, AUDIO_HAL_CTRL_START);

    esp_periph_config_t periph_cfg = DEFAULT_ESP_PERIPH_SET_CONFIG();
    esp_periph_set_handle_t set = esp_periph_set_init(&periph_cfg);
    audio_board_sdcard_init(set, SD_MODE_1_LINE);

    audio_board_key_init(set);

    input_key_service_info_t input_key_info[] = INPUT_KEY_DEFAULT_INFO();
    input_key_service_cfg_t input_cfg = INPUT_KEY_SERVICE_DEFAULT_CONFIG();
    input_cfg.handle = set;
    periph_service_handle_t input_ser = input_key_service_create(&input_cfg);
    input_key_service_add_key(input_ser, input_key_info, INPUT_KEY_NUM);
    //periph_service_set_callback(input_ser, idle_adkey_cb, NULL);

    ESP_LOGI(TAG, "[3.0] Create pipeline_mix pipeline");
    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
    audio_pipeline_handle_t pipeline_mix = audio_pipeline_init(&pipeline_cfg);

    ESP_LOGI(TAG, "[3.1] Create down-mixer element");
    downmix_cfg_t downmix_cfg = DEFAULT_DOWNMIX_CONFIG();
    downmix_cfg.downmix_info.source_num = NUMBER_SOURCE_FILE;
    downmix_cfg.downmix_info.out_ctx = ESP_DOWNMIX_OUT_CTX_NORMAL;
    downmix_cfg.downmix_info.output_type = ESP_DOWNMIX_OUTPUT_TYPE_TWO_CHANNEL;

    audio_element_handle_t downmixer = downmix_init(&downmix_cfg);
    downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM);
    downmix_set_input_rb_timeout(downmixer, 0, INDEX_NEWCOME_STREAM);

    esp_downmix_input_info_t source_information[NUMBER_SOURCE_FILE] = {0};
    esp_downmix_input_info_t source_info_base = {
        .samplerate = SAMPLERATE,
        .channel = NUM_INPUT_CHANNEL,
        .bits_num = 16,        
        .gain = {0, -10}, /* base music depress form 0dB to -10dB */
        .transit_time = TRANSMITTIME,
    };
    source_information[0] = source_info_base;

    esp_downmix_input_info_t source_info_newcome = {
        .samplerate = SAMPLERATE,
        .channel = NUM_INPUT_CHANNEL,
        .bits_num = 16,       
        .gain = {-10, 0}, /* newcome music rise form -10dB to 0dB */
        .transit_time = TRANSMITTIME,
    };
    source_information[1] = source_info_newcome;
    source_info_init(downmixer, source_information);

    ESP_LOGI(TAG, "[3.2] Create i2s stream to read audio data from codec chip");
    i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();//44100
    i2s_cfg.i2s_config.sample_rate = 44100;
    //i2s_cfg.i2s_config.dma_buf_count = 3, 
    i2s_cfg.type = I2S_MODE_MASTER | I2S_MODE_TX;
    i2s_cfg.type = AUDIO_STREAM_WRITER;

    audio_element_handle_t i2s_writer = i2s_stream_init(&i2s_cfg);

    ESP_LOGI(TAG, "[3.3] Link elements together downmixer-->i2s_writer");
    audio_pipeline_register(pipeline_mix, downmixer, "mixer");
    audio_pipeline_register(pipeline_mix, i2s_writer, "i2s");

    ESP_LOGI(TAG, "[3.4] Link elements together downmixer-->i2s_stream-->[codec_chip]");
    const char *link_mix[2] = {"mixer", "i2s"};
    audio_pipeline_link(pipeline_mix, &link_mix[0], 2);

    ESP_LOGI(TAG, "[4.0] Create Fatfs stream to read input data");
    fatfs_stream_cfg_t fatfs_cfg = FATFS_STREAM_CFG_DEFAULT();
    fatfs_cfg.type = AUDIO_STREAM_READER;
    base_fatfs_reader_el = fatfs_stream_init(&fatfs_cfg);
    audio_element_set_uri(base_fatfs_reader_el, "/sdcard/PLAYER/K0015/CP01.mp3");
    newcome_fatfs_reader_el = fatfs_stream_init(&fatfs_cfg);
    audio_element_set_uri(newcome_fatfs_reader_el, "/sdcard/UI/Success.mp3");

    ESP_LOGI(TAG, "[4.1] Create mp3 decoder to decode mp3 file");
    mp3_decoder_cfg_t mp3_cfg = DEFAULT_MP3_DECODER_CONFIG();
    base_mp3_decoder_el = mp3_decoder_init(&mp3_cfg);
    newcome_mp3_decoder_el = mp3_decoder_init(&mp3_cfg);

    // ESP_LOGI(TAG, "[4.1] Create resample element");
    // rsp_filter_cfg_t rsp_sdcard_cfg = DEFAULT_RESAMPLE_FILTER_CONFIG();
    // rsp_sdcard_cfg.src_rate = 44100;
    // rsp_sdcard_cfg.src_ch = 2;
    // rsp_sdcard_cfg.dest_rate = 44100;
    // rsp_sdcard_cfg.dest_ch = 2;
    // base_rsp_filter_el = rsp_filter_init(&rsp_sdcard_cfg);

    // rsp_sdcard_cfg.src_rate = 44100;
    // rsp_sdcard_cfg.src_ch = 2;
    // rsp_sdcard_cfg.dest_rate = 44100;
    // rsp_sdcard_cfg.dest_ch = 2;
    // newcome_rsp_filter_el = rsp_filter_init(&rsp_sdcard_cfg);

    ESP_LOGI(TAG, "[4.2] Create raw stream of base mp3 to write data");
    raw_stream_cfg_t raw_cfg = RAW_STREAM_CFG_DEFAULT();
    raw_cfg.type = AUDIO_STREAM_WRITER;
    base_raw_write_el = raw_stream_init(&raw_cfg);
    newcome_raw_write_el = raw_stream_init(&raw_cfg);

    ESP_LOGI(TAG, "[5.0] Set up  event listener");
    audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
    audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg);

    audio_pipeline_handle_t base_stream_pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(base_stream_pipeline);
    audio_pipeline_register(base_stream_pipeline, base_fatfs_reader_el, "base_file");
    audio_pipeline_register(base_stream_pipeline, base_mp3_decoder_el, "base_mp3");
    //audio_pipeline_register(base_stream_pipeline, base_rsp_filter_el, "base_filter");
    audio_pipeline_register(base_stream_pipeline, base_raw_write_el, "base_raw");

    //const char *link_tag_base[4] = {"base_file", "base_mp3", "base_filter", "base_raw"};
    //audio_pipeline_link(base_stream_pipeline, &link_tag_base[0], 4);

    const char *link_tag_base[3] = {"base_file", "base_mp3", "base_raw"};
    audio_pipeline_link(base_stream_pipeline, &link_tag_base[0], 3);

    ringbuf_handle_t rb_base = audio_element_get_input_ringbuf(base_raw_write_el);
    downmix_set_input_rb(downmixer, rb_base, 0);
    audio_pipeline_set_listener(base_stream_pipeline, evt);

    audio_pipeline_handle_t newcome_stream_pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(newcome_stream_pipeline);
    audio_pipeline_register(newcome_stream_pipeline, newcome_fatfs_reader_el, "newcome_file");
    audio_pipeline_register(newcome_stream_pipeline, newcome_mp3_decoder_el, "newcome_mp3");
    //audio_pipeline_register(newcome_stream_pipeline, newcome_rsp_filter_el, "newcome_filter");
    audio_pipeline_register(newcome_stream_pipeline, newcome_raw_write_el, "newcome_raw");

    //const char *link_tag_newcome[4] = {"newcome_file", "newcome_mp3", "newcome_filter", "newcome_raw"};
    //audio_pipeline_link(newcome_stream_pipeline, &link_tag_newcome[0], 4);

    const char *link_tag_newcome[3] = {"newcome_file", "newcome_mp3", "newcome_raw"};
    audio_pipeline_link(newcome_stream_pipeline, &link_tag_newcome[0], 3);

    ringbuf_handle_t rb_newcome = audio_element_get_input_ringbuf(newcome_raw_write_el);
    downmix_set_input_rb(downmixer, rb_newcome, 1);
    audio_pipeline_set_listener(newcome_stream_pipeline, evt);

    ESP_LOGI(TAG, "[5.1] Listening event from peripherals");
    audio_pipeline_set_listener(pipeline_mix, evt);
    audio_event_iface_set_listener(esp_periph_set_get_event_iface(set), evt);
    downmix_set_output_type(downmixer, PLAY_STATUS);

    audio_pipeline_run(base_stream_pipeline);
    audio_pipeline_run(pipeline_mix);
    downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_BYPASS);
    ESP_LOGI(TAG, "[6.0] Base stream pipeline running");

    i2s_stream_set_clk(i2s_writer, SAMPLERATE, 16, PLAY_STATUS);
    audio_hal_set_volume(board_handle->audio_hal, 40); 

    while (1) {
        audio_event_iface_msg_t msg;
        esp_err_t ret = audio_event_iface_listen(evt, &msg, portMAX_DELAY);
        if (ret != ESP_OK) {
            ESP_LOGE(TAG, "[ * ] Event interface error : %d", ret);
            continue;
        }

        if (((int)msg.data == get_input_mode_id()) && (msg.cmd == PERIPH_BUTTON_PRESSED)) {

            audio_pipeline_run(newcome_stream_pipeline);
            downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_ON);
            downmix_set_input_rb_timeout(downmixer, 50, INDEX_BASE_STREAM);
            downmix_set_input_rb_timeout(downmixer, 50, INDEX_NEWCOME_STREAM);
            ESP_LOGI(TAG, "TONE running...");
        }

        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && 
            msg.source == (void *)i2s_writer && 
            msg.cmd == AEL_MSG_CMD_REPORT_STATUS && 
            (((int)msg.data == AEL_STATUS_STATE_STOPPED) || 
            ((int)msg.data == AEL_STATUS_STATE_FINISHED))){

            ESP_LOGW(TAG, "Player finsihed, break loop");
            break;
        }

        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && 
            msg.source == (void *)newcome_mp3_decoder_el/*newcome_rsp_filter_el*/ && 
            msg.cmd == AEL_MSG_CMD_REPORT_STATUS && 
            (((int) msg.data == AEL_STATUS_STATE_STOPPED) || 
            ((int)msg.data == AEL_STATUS_STATE_FINISHED))){

            downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_OFF);
            downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM);
            downmix_set_input_rb_timeout(downmixer, 0, INDEX_NEWCOME_STREAM);
            audio_pipeline_stop(newcome_stream_pipeline);
            audio_pipeline_wait_for_stop(newcome_stream_pipeline);
            audio_pipeline_terminate(newcome_stream_pipeline);
            audio_pipeline_reset_ringbuffer(newcome_stream_pipeline);
            audio_pipeline_reset_elements(newcome_stream_pipeline);
            ESP_LOGI(TAG, "TONE finsihed");
        }
    }

    ESP_LOGI(TAG, "[7.0] Stop all pipelines");
    /* Stop base stream pipeline, Release resources */
    audio_pipeline_stop(base_stream_pipeline);
    audio_pipeline_wait_for_stop(base_stream_pipeline);
    audio_pipeline_terminate(base_stream_pipeline);
    audio_pipeline_unregister_more( base_stream_pipeline, 
                                    base_fatfs_reader_el,
                                    base_mp3_decoder_el, 
                                    //base_rsp_filter_el, 
                                    base_raw_write_el, NULL);
    audio_pipeline_remove_listener(base_stream_pipeline);
    audio_pipeline_deinit(base_stream_pipeline);
    audio_element_deinit(base_fatfs_reader_el);
    audio_element_deinit(base_mp3_decoder_el);
    //audio_element_deinit(base_rsp_filter_el);
    audio_element_deinit(base_raw_write_el);

    /* Stop newcome stream pipeline, Release resources */
    audio_pipeline_stop(newcome_stream_pipeline);
    audio_pipeline_wait_for_stop(newcome_stream_pipeline);
    audio_pipeline_terminate(newcome_stream_pipeline);
    audio_pipeline_unregister_more( newcome_stream_pipeline, 
                                    newcome_fatfs_reader_el,
                                    newcome_mp3_decoder_el, 
                                    //newcome_rsp_filter_el, 
                                    newcome_raw_write_el, NULL);
    audio_pipeline_remove_listener(newcome_stream_pipeline);
    audio_pipeline_deinit(newcome_stream_pipeline);
    audio_element_deinit(newcome_fatfs_reader_el);
    audio_element_deinit(newcome_mp3_decoder_el);
    //audio_element_deinit(newcome_rsp_filter_el);
    audio_element_deinit(newcome_raw_write_el);

    /* Stop mixer stream pipeline, Release resources */
    audio_pipeline_stop(pipeline_mix);
    audio_pipeline_wait_for_stop(pipeline_mix);
    audio_pipeline_terminate(pipeline_mix);
    audio_pipeline_unregister_more(pipeline_mix, downmixer, i2s_writer, NULL);
    audio_pipeline_remove_listener(pipeline_mix);

    /* Stop all peripherals before removing the listener */
    esp_periph_set_stop_all(set);
    audio_event_iface_remove_listener(esp_periph_set_get_event_iface(set), evt);

    /* Make sure audio_pipeline_remove_listener & audio_event_iface_remove_listener are called before destroying event_iface */
    audio_event_iface_destroy(evt);

    /* Release resources */
    audio_pipeline_deinit(pipeline_mix);
    audio_element_deinit(downmixer);
    audio_element_deinit(i2s_writer);
    esp_periph_set_destroy(set);
}
crispmfc commented 1 year ago

Hello, there is an example that allows to play multiple audios simultaneously. :) https://github.com/espressif/esp-adf/tree/master/examples/advanced_examples/downmix_pipeline

0x0fe commented 1 year ago

@crispmfc nice, i missed this one

crispmfc commented 1 year ago

Hello, could you please tell me which lines of code and where did you put them to define the sdio pin of sdcard? I've been trying to set the pins for days without success. Not only that, it seems that now I have managed to hardcode the I2S pins and can no longer define them manually from my main.c (Maybe because I modified the boards that come in the examples) but hey, it would help me to know which is the correct path at least with sdcard. With i2s I don't have many problems. Thanks in advance.

0x0fe commented 1 year ago

@crispmfc you have to setup the board related files, and the best way to do this is to create a custom board, there is a topic about that in the documentation. In fact a lot of things are defined in this set of files, including your codec settings, so it is very important to do it properly. Here is a screenshot of the custom board definition files for this specific project:

image image image

crispmfc commented 1 year ago

Thank you, I really appreciate it! I will follow your instructions, since I have been trying solutions for many hours for about 1 week and precisely the custom board is something that I need to do.

0x0fe commented 1 year ago

@crispmfc OK, you can use this as a base https://github.com/espressif/esp-adf/tree/master/examples/get-started/play_mp3_control/components/my_board

jason-mao commented 12 months ago

@0x0fe Has this issue been solved?

0x0fe commented 12 months ago

hello @jason-mao No, this issue has not been solved and is very blocking on our side.

majingjing123 commented 11 months ago

Hi, could you please provide the two audio files you tested? I have already test your example with ADF v2.6 and IDF v5.0.2, but this issue did not recur.

0x0fe commented 11 months ago

@majingjing123 please find the files attached, CP01.mp3 is the playback asset, Success.mp3 is the "tone". CP01.zip Success.zip

and here is my SDKconfig in case it has anything to do with it: sdkconfig.zip

The exact firmware used for test is below

Test Firmware ```cpp #if 1 #include #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "freertos/event_groups.h" #include "esp_log.h" #include "audio_element.h" #include "audio_pipeline.h" #include "audio_event_iface.h" #include "i2s_stream.h" #include "mp3_decoder.h" #include "fatfs_stream.h" #include "downmix.h" #include "filter_resample.h" #include "raw_stream.h" #include "board.h" #include "periph_sdcard.h" #include "periph_button.h" #include "input_key_service.h" #include "periph_adc_button.h" static const char *TAG = "DOWNMIX_PIPELINE_EXAMPLE"; #define INDEX_BASE_STREAM 0 #define INDEX_NEWCOME_STREAM 1 #define SAMPLERATE 44100 #define NUM_INPUT_CHANNEL 2 #define TRANSMITTIME 500 #define MUSIC_GAIN_DB 0 #define PLAY_STATUS ESP_DOWNMIX_OUTPUT_TYPE_TWO_CHANNEL //ESP_DOWNMIX_OUTPUT_TYPE_ONE_CHANNEL #define NUMBER_SOURCE_FILE 2 void app_main(void) { audio_element_handle_t base_fatfs_reader_el = NULL; audio_element_handle_t base_mp3_decoder_el = NULL; //audio_element_handle_t base_rsp_filter_el = NULL; audio_element_handle_t base_raw_write_el = NULL; audio_element_handle_t newcome_fatfs_reader_el = NULL; audio_element_handle_t newcome_mp3_decoder_el = NULL; //audio_element_handle_t newcome_rsp_filter_el = NULL; audio_element_handle_t newcome_raw_write_el = NULL; esp_log_level_set("*", ESP_LOG_WARN); esp_log_level_set(TAG, ESP_LOG_INFO); audio_board_handle_t board_handle = audio_board_init(); audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_DECODE, AUDIO_HAL_CTRL_START); esp_periph_config_t periph_cfg = DEFAULT_ESP_PERIPH_SET_CONFIG(); esp_periph_set_handle_t set = esp_periph_set_init(&periph_cfg); audio_board_sdcard_init(set, SD_MODE_1_LINE); audio_board_key_init(set); input_key_service_info_t input_key_info[] = INPUT_KEY_DEFAULT_INFO(); input_key_service_cfg_t input_cfg = INPUT_KEY_SERVICE_DEFAULT_CONFIG(); input_cfg.handle = set; periph_service_handle_t input_ser = input_key_service_create(&input_cfg); input_key_service_add_key(input_ser, input_key_info, INPUT_KEY_NUM); //periph_service_set_callback(input_ser, idle_adkey_cb, NULL); ESP_LOGI(TAG, "[3.0] Create pipeline_mix pipeline"); audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG(); audio_pipeline_handle_t pipeline_mix = audio_pipeline_init(&pipeline_cfg); ESP_LOGI(TAG, "[3.1] Create down-mixer element"); downmix_cfg_t downmix_cfg = DEFAULT_DOWNMIX_CONFIG(); downmix_cfg.downmix_info.source_num = NUMBER_SOURCE_FILE; downmix_cfg.downmix_info.out_ctx = ESP_DOWNMIX_OUT_CTX_LEFT_RIGHT; downmix_cfg.downmix_info.output_type = ESP_DOWNMIX_OUTPUT_TYPE_TWO_CHANNEL; //downmix_cfg.task_prio = 22; //downmix_cfg.task_core = 1; audio_element_handle_t downmixer = downmix_init(&downmix_cfg); downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM); downmix_set_input_rb_timeout(downmixer, 0, INDEX_NEWCOME_STREAM); esp_downmix_input_info_t source_information[NUMBER_SOURCE_FILE] = {0}; esp_downmix_input_info_t source_info_base = { .samplerate = SAMPLERATE, .channel = NUM_INPUT_CHANNEL, .bits_num = 16, .gain = {0, -10}, /* base music depress form 0dB to -10dB */ .transit_time = TRANSMITTIME, }; source_information[0] = source_info_base; esp_downmix_input_info_t source_info_newcome = { .samplerate = SAMPLERATE, .channel = NUM_INPUT_CHANNEL, .bits_num = 16, .gain = {-10, 0}, /* newcome music rise form -10dB to 0dB */ .transit_time = TRANSMITTIME, }; source_information[1] = source_info_newcome; source_info_init(downmixer, source_information); ESP_LOGI(TAG, "[3.2] Create i2s stream to read audio data from codec chip"); i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();//44100 i2s_cfg.type = AUDIO_STREAM_WRITER; i2s_cfg.i2s_config.mode = (i2s_mode_t)(I2S_MODE_MASTER | I2S_MODE_TX); //i2s_cfg.i2s_config.sample_rate = 44100; //i2s_cfg.i2s_config.dma_buf_count = 3; //i2s_cfg.i2s_config.dma_buf_len = 1024; //i2s_cfg.i2s_config.use_apll = false; //i2s_cfg.i2s_config.tx_desc_auto_clear = true; //i2s_cfg.i2s_config.fixed_mclk = 0 ; audio_element_handle_t i2s_writer = i2s_stream_init(&i2s_cfg); ESP_LOGI(TAG, "[3.3] Link elements together downmixer-->i2s_writer"); audio_pipeline_register(pipeline_mix, downmixer, "mixer"); audio_pipeline_register(pipeline_mix, i2s_writer, "i2s"); ESP_LOGI(TAG, "[3.4] Link elements together downmixer-->i2s_stream-->[codec_chip]"); const char *link_mix[2] = {"mixer", "i2s"}; audio_pipeline_link(pipeline_mix, &link_mix[0], 2); ESP_LOGI(TAG, "[4.0] Create Fatfs stream to read input data"); fatfs_stream_cfg_t fatfs_cfg = FATFS_STREAM_CFG_DEFAULT(); fatfs_cfg.type = AUDIO_STREAM_READER; base_fatfs_reader_el = fatfs_stream_init(&fatfs_cfg); audio_element_set_uri(base_fatfs_reader_el, "/sdcard/PLAYER/K0015/CP01.mp3"); newcome_fatfs_reader_el = fatfs_stream_init(&fatfs_cfg); audio_element_set_uri(newcome_fatfs_reader_el, "/sdcard/UI/Success.mp3"); ESP_LOGI(TAG, "[4.1] Create mp3 decoder to decode mp3 file"); mp3_decoder_cfg_t mp3_cfg = DEFAULT_MP3_DECODER_CONFIG(); base_mp3_decoder_el = mp3_decoder_init(&mp3_cfg); newcome_mp3_decoder_el = mp3_decoder_init(&mp3_cfg); // ESP_LOGI(TAG, "[4.1] Create resample element"); // rsp_filter_cfg_t rsp_sdcard_cfg = DEFAULT_RESAMPLE_FILTER_CONFIG(); // rsp_sdcard_cfg.src_rate = 44100; // rsp_sdcard_cfg.src_ch = 2; // rsp_sdcard_cfg.dest_rate = 44100; // rsp_sdcard_cfg.dest_ch = 2; // base_rsp_filter_el = rsp_filter_init(&rsp_sdcard_cfg); // rsp_sdcard_cfg.src_rate = 44100; // rsp_sdcard_cfg.src_ch = 2; // rsp_sdcard_cfg.dest_rate = 44100; // rsp_sdcard_cfg.dest_ch = 2; // newcome_rsp_filter_el = rsp_filter_init(&rsp_sdcard_cfg); ESP_LOGI(TAG, "[4.2] Create raw stream of base mp3 to write data"); raw_stream_cfg_t raw_cfg = RAW_STREAM_CFG_DEFAULT(); raw_cfg.type = AUDIO_STREAM_WRITER; base_raw_write_el = raw_stream_init(&raw_cfg); newcome_raw_write_el = raw_stream_init(&raw_cfg); ESP_LOGI(TAG, "[5.0] Set up event listener"); audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG(); audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg); audio_pipeline_handle_t base_stream_pipeline = audio_pipeline_init(&pipeline_cfg); mem_assert(base_stream_pipeline); audio_pipeline_register(base_stream_pipeline, base_fatfs_reader_el, "base_file"); audio_pipeline_register(base_stream_pipeline, base_mp3_decoder_el, "base_mp3"); //audio_pipeline_register(base_stream_pipeline, base_rsp_filter_el, "base_filter"); audio_pipeline_register(base_stream_pipeline, base_raw_write_el, "base_raw"); //const char *link_tag_base[4] = {"base_file", "base_mp3", "base_filter", "base_raw"}; //audio_pipeline_link(base_stream_pipeline, &link_tag_base[0], 4); const char *link_tag_base[3] = {"base_file", "base_mp3", "base_raw"}; audio_pipeline_link(base_stream_pipeline, &link_tag_base[0], 3); ringbuf_handle_t rb_base = audio_element_get_input_ringbuf(base_raw_write_el); downmix_set_input_rb(downmixer, rb_base, 0); audio_pipeline_set_listener(base_stream_pipeline, evt); audio_pipeline_handle_t newcome_stream_pipeline = audio_pipeline_init(&pipeline_cfg); mem_assert(newcome_stream_pipeline); audio_pipeline_register(newcome_stream_pipeline, newcome_fatfs_reader_el, "newcome_file"); audio_pipeline_register(newcome_stream_pipeline, newcome_mp3_decoder_el, "newcome_mp3"); //audio_pipeline_register(newcome_stream_pipeline, newcome_rsp_filter_el, "newcome_filter"); audio_pipeline_register(newcome_stream_pipeline, newcome_raw_write_el, "newcome_raw"); //const char *link_tag_newcome[4] = {"newcome_file", "newcome_mp3", "newcome_filter", "newcome_raw"}; //audio_pipeline_link(newcome_stream_pipeline, &link_tag_newcome[0], 4); const char *link_tag_newcome[3] = {"newcome_file", "newcome_mp3", "newcome_raw"}; audio_pipeline_link(newcome_stream_pipeline, &link_tag_newcome[0], 3); ringbuf_handle_t rb_newcome = audio_element_get_input_ringbuf(newcome_raw_write_el); downmix_set_input_rb(downmixer, rb_newcome, 1); audio_pipeline_set_listener(newcome_stream_pipeline, evt); ESP_LOGI(TAG, "[5.1] Listening event from peripherals"); audio_pipeline_set_listener(pipeline_mix, evt); audio_event_iface_set_listener(esp_periph_set_get_event_iface(set), evt); downmix_set_output_type(downmixer, PLAY_STATUS); audio_pipeline_run(base_stream_pipeline); audio_pipeline_run(pipeline_mix); downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_BYPASS); ESP_LOGI(TAG, "[6.0] Base stream pipeline running"); i2s_stream_set_clk(i2s_writer, SAMPLERATE, 16, PLAY_STATUS); audio_hal_set_volume(board_handle->audio_hal, 40); //downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_ON); //downmix_set_input_rb_timeout(downmixer, 50, INDEX_BASE_STREAM); //downmix_set_input_rb_timeout(downmixer, 50, INDEX_NEWCOME_STREAM); while (1) { audio_event_iface_msg_t msg; esp_err_t ret = audio_event_iface_listen(evt, &msg, portMAX_DELAY); if (ret != ESP_OK) { ESP_LOGE(TAG, "[ * ] Event interface error : %d", ret); continue; } if (((int)msg.data == get_input_mode_id()) && (msg.cmd == PERIPH_BUTTON_PRESSED)) { audio_pipeline_run(newcome_stream_pipeline); downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_ON); downmix_set_input_rb_timeout(downmixer, 50, INDEX_BASE_STREAM); downmix_set_input_rb_timeout(downmixer, 50, INDEX_NEWCOME_STREAM); ESP_LOGI(TAG, "TONE running..."); } if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && msg.source == (void *)i2s_writer && msg.cmd == AEL_MSG_CMD_REPORT_STATUS && (((int)msg.data == AEL_STATUS_STATE_STOPPED) || ((int)msg.data == AEL_STATUS_STATE_FINISHED))){ ESP_LOGW(TAG, "Player finsihed, break loop"); break; } if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && msg.source == (void *)newcome_mp3_decoder_el/*newcome_rsp_filter_el*/ && msg.cmd == AEL_MSG_CMD_REPORT_STATUS && (((int) msg.data == AEL_STATUS_STATE_STOPPED) || ((int)msg.data == AEL_STATUS_STATE_FINISHED))){ downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_OFF); downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM); downmix_set_input_rb_timeout(downmixer, 0, INDEX_NEWCOME_STREAM); audio_pipeline_stop(newcome_stream_pipeline); audio_pipeline_wait_for_stop(newcome_stream_pipeline); audio_pipeline_terminate(newcome_stream_pipeline); audio_pipeline_reset_ringbuffer(newcome_stream_pipeline); audio_pipeline_reset_elements(newcome_stream_pipeline); ESP_LOGI(TAG, "TONE finsihed"); } } ESP_LOGI(TAG, "[7.0] Stop all pipelines"); /* Stop base stream pipeline, Release resources */ audio_pipeline_stop(base_stream_pipeline); audio_pipeline_wait_for_stop(base_stream_pipeline); audio_pipeline_terminate(base_stream_pipeline); audio_pipeline_unregister_more( base_stream_pipeline, base_fatfs_reader_el, base_mp3_decoder_el, //base_rsp_filter_el, base_raw_write_el, NULL); audio_pipeline_remove_listener(base_stream_pipeline); audio_pipeline_deinit(base_stream_pipeline); audio_element_deinit(base_fatfs_reader_el); audio_element_deinit(base_mp3_decoder_el); //audio_element_deinit(base_rsp_filter_el); audio_element_deinit(base_raw_write_el); /* Stop newcome stream pipeline, Release resources */ audio_pipeline_stop(newcome_stream_pipeline); audio_pipeline_wait_for_stop(newcome_stream_pipeline); audio_pipeline_terminate(newcome_stream_pipeline); audio_pipeline_unregister_more( newcome_stream_pipeline, newcome_fatfs_reader_el, newcome_mp3_decoder_el, //newcome_rsp_filter_el, newcome_raw_write_el, NULL); audio_pipeline_remove_listener(newcome_stream_pipeline); audio_pipeline_deinit(newcome_stream_pipeline); audio_element_deinit(newcome_fatfs_reader_el); audio_element_deinit(newcome_mp3_decoder_el); //audio_element_deinit(newcome_rsp_filter_el); audio_element_deinit(newcome_raw_write_el); /* Stop mixer stream pipeline, Release resources */ audio_pipeline_stop(pipeline_mix); audio_pipeline_wait_for_stop(pipeline_mix); audio_pipeline_terminate(pipeline_mix); audio_pipeline_unregister_more(pipeline_mix, downmixer, i2s_writer, NULL); audio_pipeline_remove_listener(pipeline_mix); /* Stop all peripherals before removing the listener */ esp_periph_set_stop_all(set); audio_event_iface_remove_listener(esp_periph_set_get_event_iface(set), evt); /* Make sure audio_pipeline_remove_listener & audio_event_iface_remove_listener are called before destroying event_iface */ audio_event_iface_destroy(evt); /* Release resources */ audio_pipeline_deinit(pipeline_mix); audio_element_deinit(downmixer); audio_element_deinit(i2s_writer); esp_periph_set_destroy(set); } #endif ```
majingjing123 commented 11 months ago

@0x0fe Hi, i have already use your test music and sdkconfig to test example on ESP32 with board ESP32_LyraT, but unfortunately, i still fail to reproduce this issue. Here is my test bin. You can test it. downmix.zip And could you please provide your test bin to help me reproduce the issue.

0x0fe commented 11 months ago

@majingjing123 I cannot test your BIN because the baord has a different codec. No problem to provide a bin to you, but you also wont have the same codec, so i am not sure how it will be useful. My codec is NAU88C22. Also isnt the LyraT codec fixed 48KHz? my code disable all sample rate conversion, so how can it run normally at 44.1KHz?

majingjing123 commented 11 months ago

@0x0fe Hi, our LyraT codec can play all the sample rate. Do you have our official development board， such as 'ESP32_LyraT' or 'ESP32-S3-Korvo'. You can use our development board to test the issues. If can't recur this issue, you could check for hardware issues.

0x0fe commented 11 months ago

@majingjing123 Oh ok i see, no i dont have official dev board, only the custom design. I dont see how it could come from the hardware, since it plays fine in all scenario (sd_mmc, https, a2dp sink etc. In fact on the downmix example when both sources are working together it plays fine, the problem occurs when the playback assets plays alone. Which log should i enable to get more clue on the issue? Also note, we use ESP32D0WDR2, which has internal 2M PSRAM

majingjing123 commented 11 months ago

@0x0fe I would like to confirm that the ADF branch is https://github.com/espressif/esp-adf/releases, and the IDF branch is `commit 5181de8ac5ec5e18f04f634da8ce173b7ef5ab73 (HEAD, tag: v5.0.2) Author: Ivan Grokhotkov ivan@espressif.com Date: Wed May 10 09:59:18 2023 +0200

versions: Update version to 5.0.2

`

0x0fe commented 11 months ago

@majingjing123 Hi,

The ADF branch is : https://github.com/espressif/esp-adf/archive/refs/tags/v2.6.zip The IDF branch yes, it is framework-espidf @ 3.50002.230601 which corresponds to 5.0.2 https://github.com/espressif/esp-idf/commit/5181de8ac5ec5e18f04f634da8ce173b7ef5ab73 : 

PACKAGES:
 - framework-espidf @ 3.50002.230601 (5.0.2)
 - tool-idf @ 1.0.1
 - tool-mconf @ 1.4060000.20190628 (406.0.0)
 - tool-ninja @ 1.9.0
 - toolchain-esp32ulp @ 1.23500.220830 (2.35.0)
 - toolchain-xtensa-esp32 @ 11.2.0+2022r1

Could the issue be related to mclk? my codec uses (and needs) mclk, i use GPIO3 for this purpose. My board_pin_config.c is correctly set, and the I2S works perfectly fine in all other scenarios i tested, but i wonder wether the downmix could interfere in some way with the mclk.

I ran more tests and i can confirm the problem always occurs when both pipeline are set but only one is running. As soon as both are running at the same time everything is perfect, which excludes an issue with sd_mmc for example. I tried to enable fixed_mclk and/or disable apll, but it did not change anything.

I recorded a sample of the issue (with a mobile, from the device tiny speaker), the background file is the same already provided here, i replaced the "tone" file by a longer sample so there is enough time to hear that the playback is normal when both the background and the "tone" files are playing together. downmix_issue.zip

By the way is there a way to ensure the default UART port (the logging port) is set to simplex mode (TX only) because as told, i use GPIO3 for mclk and so i would like that the log uart is cleanly set to TX only at the setup.

majingjing123 commented 11 months ago

@0x0fe Hi,i have played your issue music, but the sample rate is 48kHz. The music you created for the project should be at 44.1Khz. Please check it. You'd better to write the downmix data to file. You can also try to change the downmix_set_input_rb_timeout, don't set to 0, try to set to 10.

0x0fe commented 11 months ago

@majingjing123 There must be a confusion, the sample i sent was not destined to be played on the board, it was destined to be played on a PC or mobile to hear the issue, so the samplerate is not relevant. OK i will change the downmix_set_input_rb_timeout and see if it helps.

0x0fe commented 11 months ago

@majingjing123 Well, setting downmix_set_input_rb_timeout solved the issue. Finally, thank you for mentioning this parameter.

Is there any "player" wrapper where the downmix is implemented for tone playback? Because the example takes 350 lines to just play a file over another, which is quite extreme.

majingjing123 commented 11 months ago

@0x0fe Our player currently does not support downmix. You need to integrate this feature into your project by yourself. This example only demonstrate how downmix work.

0x0fe commented 11 months ago

@majingjing123 Unfortunately there is now a new problem, while the base playback works fine, and it also works fine when both the base playback and newcome are playing together, as soon as the newcomes finises, the base gets messed up again. I will take a small recording. I tried to set the downmix_set_input_rb_timeout to 0, 10 and 50, it does not help.

image

Recording of the new issue :

Downmix issue 2.zip

majingjing123 commented 11 months ago

@0x0fe You can comment downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM); out， because base stream is still playing, but you set its timeout to 0. If the stream is playing, you'd better don't set its timeout to 0.

0x0fe commented 11 months ago

@majingjing123 No, it doesnt work. Commenting downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM); totally cuts all audio when the newcome is finished commenting both 

downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM);
downmix_set_input_rb_timeout(downmixer, 0, INDEX_NEWCOME_STREAM);

leads to massive samplerate issue, similar to before but 10x worst. I also tried with timeout at 10 and 50 before, it did not solve the issue.

if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && 
            msg.source == (void *)newcome_mp3_decoder_el/*newcome_rsp_filter_el*/ && 
            msg.cmd == AEL_MSG_CMD_REPORT_STATUS && 
            (((int) msg.data == AEL_STATUS_STATE_STOPPED) || 
            ((int)msg.data == AEL_STATUS_STATE_FINISHED))){

            downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_OFF);
            //downmix_set_input_rb_timeout(downmixer, 10, INDEX_BASE_STREAM);
            downmix_set_input_rb_timeout(downmixer, 10, INDEX_NEWCOME_STREAM);
            audio_pipeline_stop(newcome_stream_pipeline);
            audio_pipeline_wait_for_stop(newcome_stream_pipeline);
            audio_pipeline_terminate(newcome_stream_pipeline);
            audio_pipeline_reset_ringbuffer(newcome_stream_pipeline);
            audio_pipeline_reset_elements(newcome_stream_pipeline);
            ESP_LOGI(TAG, "TONE finsihed");
}

Dow mix issue 3.zip

majingjing123 commented 11 months ago

@0x0fe Try this code.

`#include

include "freertos/FreeRTOS.h"

include "freertos/task.h"

include "freertos/event_groups.h"

include "esp_log.h"

include "audio_element.h"

include "audio_pipeline.h"

include "audio_event_iface.h"

include "i2s_stream.h"

include "mp3_decoder.h"

include "fatfs_stream.h"

include "downmix.h"

include "filter_resample.h"

include "raw_stream.h"

include "board.h"

include "periph_sdcard.h"

include "periph_button.h"

include "input_key_service.h"

include "periph_adc_button.h"

static const char *TAG = "DOWNMIX_PIPELINE_EXAMPLE";

define INDEX_BASE_STREAM 0

define INDEX_NEWCOME_STREAM 1

define SAMPLERATE 44100

define NUM_INPUT_CHANNEL 2

define TRANSMITTIME 500

define MUSIC_GAIN_DB 0

define PLAY_STATUS ESP_DOWNMIX_OUTPUT_TYPE_TWO_CHANNEL //ESP_DOWNMIX_OUTPUT_TYPE_ONE_CHANNEL

define NUMBER_SOURCE_FILE 2

void app_main(void) { audio_element_handle_t base_fatfs_reader_el = NULL; audio_element_handle_t base_mp3_decoder_el = NULL; //audio_element_handle_t base_rsp_filter_el = NULL; audio_element_handle_t base_raw_write_el = NULL;

audio_element_handle_t newcome_fatfs_reader_el = NULL;
audio_element_handle_t newcome_mp3_decoder_el = NULL;
//audio_element_handle_t newcome_rsp_filter_el = NULL;
audio_element_handle_t newcome_raw_write_el = NULL;

esp_log_level_set("*", ESP_LOG_WARN);
esp_log_level_set(TAG, ESP_LOG_INFO);

audio_board_handle_t board_handle = audio_board_init();
audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_DECODE, AUDIO_HAL_CTRL_START);

esp_periph_config_t periph_cfg = DEFAULT_ESP_PERIPH_SET_CONFIG();
esp_periph_set_handle_t set = esp_periph_set_init(&periph_cfg);
audio_board_sdcard_init(set, SD_MODE_1_LINE);

audio_board_key_init(set);

input_key_service_info_t input_key_info[] = INPUT_KEY_DEFAULT_INFO();
input_key_service_cfg_t input_cfg = INPUT_KEY_SERVICE_DEFAULT_CONFIG();
input_cfg.handle = set;
periph_service_handle_t input_ser = input_key_service_create(&input_cfg);
input_key_service_add_key(input_ser, input_key_info, INPUT_KEY_NUM);
//periph_service_set_callback(input_ser, idle_adkey_cb, NULL);

ESP_LOGI(TAG, "[3.0] Create pipeline_mix pipeline");
audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
audio_pipeline_handle_t pipeline_mix = audio_pipeline_init(&pipeline_cfg);

ESP_LOGI(TAG, "[3.1] Create down-mixer element");
downmix_cfg_t downmix_cfg = DEFAULT_DOWNMIX_CONFIG();
downmix_cfg.downmix_info.source_num = NUMBER_SOURCE_FILE;
downmix_cfg.downmix_info.out_ctx = ESP_DOWNMIX_OUT_CTX_NORMAL;
downmix_cfg.downmix_info.output_type = ESP_DOWNMIX_OUTPUT_TYPE_TWO_CHANNEL;

audio_element_handle_t downmixer = downmix_init(&downmix_cfg);
downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM);
downmix_set_input_rb_timeout(downmixer, 0, INDEX_NEWCOME_STREAM);

esp_downmix_input_info_t source_information[NUMBER_SOURCE_FILE] = {0};
esp_downmix_input_info_t source_info_base = {
    .samplerate = SAMPLERATE,
    .channel = NUM_INPUT_CHANNEL,
    .bits_num = 16,        
    .gain = {0, -10}, /* base music depress form 0dB to -10dB */
    .transit_time = TRANSMITTIME,
};
source_information[0] = source_info_base;

esp_downmix_input_info_t source_info_newcome = {
    .samplerate = SAMPLERATE,
    .channel = NUM_INPUT_CHANNEL,
    .bits_num = 16,       
    .gain = {-10, 0}, /* newcome music rise form -10dB to 0dB */
    .transit_time = TRANSMITTIME,
};
source_information[1] = source_info_newcome;
source_info_init(downmixer, source_information);

ESP_LOGI(TAG, "[3.2] Create i2s stream to read audio data from codec chip");
i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();//44100
i2s_cfg.i2s_config.sample_rate = 44100;
//i2s_cfg.i2s_config.dma_buf_count = 3, 
i2s_cfg.type = I2S_MODE_MASTER | I2S_MODE_TX;
i2s_cfg.type = AUDIO_STREAM_WRITER;

audio_element_handle_t i2s_writer = i2s_stream_init(&i2s_cfg);

ESP_LOGI(TAG, "[3.3] Link elements together downmixer-->i2s_writer");
audio_pipeline_register(pipeline_mix, downmixer, "mixer");
audio_pipeline_register(pipeline_mix, i2s_writer, "i2s");

ESP_LOGI(TAG, "[3.4] Link elements together downmixer-->i2s_stream-->[codec_chip]");
const char *link_mix[2] = {"mixer", "i2s"};
audio_pipeline_link(pipeline_mix, &link_mix[0], 2);

ESP_LOGI(TAG, "[4.0] Create Fatfs stream to read input data");
fatfs_stream_cfg_t fatfs_cfg = FATFS_STREAM_CFG_DEFAULT();
fatfs_cfg.type = AUDIO_STREAM_READER;
base_fatfs_reader_el = fatfs_stream_init(&fatfs_cfg);
audio_element_set_uri(base_fatfs_reader_el, "/sdcard/PLAYER/K0015/CP01.mp3");
newcome_fatfs_reader_el = fatfs_stream_init(&fatfs_cfg);
audio_element_set_uri(newcome_fatfs_reader_el, "/sdcard/UI/Success.mp3");

ESP_LOGI(TAG, "[4.1] Create mp3 decoder to decode mp3 file");
mp3_decoder_cfg_t mp3_cfg = DEFAULT_MP3_DECODER_CONFIG();
base_mp3_decoder_el = mp3_decoder_init(&mp3_cfg);
newcome_mp3_decoder_el = mp3_decoder_init(&mp3_cfg);

// ESP_LOGI(TAG, "[4.1] Create resample element");
// rsp_filter_cfg_t rsp_sdcard_cfg = DEFAULT_RESAMPLE_FILTER_CONFIG();
// rsp_sdcard_cfg.src_rate = 44100;
// rsp_sdcard_cfg.src_ch = 2;
// rsp_sdcard_cfg.dest_rate = 44100;
// rsp_sdcard_cfg.dest_ch = 2;
// base_rsp_filter_el = rsp_filter_init(&rsp_sdcard_cfg);

// rsp_sdcard_cfg.src_rate = 44100;
// rsp_sdcard_cfg.src_ch = 2;
// rsp_sdcard_cfg.dest_rate = 44100;
// rsp_sdcard_cfg.dest_ch = 2;
// newcome_rsp_filter_el = rsp_filter_init(&rsp_sdcard_cfg);

ESP_LOGI(TAG, "[4.2] Create raw stream of base mp3 to write data");
raw_stream_cfg_t raw_cfg = RAW_STREAM_CFG_DEFAULT();
raw_cfg.type = AUDIO_STREAM_WRITER;
base_raw_write_el = raw_stream_init(&raw_cfg);
newcome_raw_write_el = raw_stream_init(&raw_cfg);

ESP_LOGI(TAG, "[5.0] Set up  event listener");
audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg);

audio_pipeline_handle_t base_stream_pipeline = audio_pipeline_init(&pipeline_cfg);
mem_assert(base_stream_pipeline);
audio_pipeline_register(base_stream_pipeline, base_fatfs_reader_el, "base_file");
audio_pipeline_register(base_stream_pipeline, base_mp3_decoder_el, "base_mp3");
//audio_pipeline_register(base_stream_pipeline, base_rsp_filter_el, "base_filter");
audio_pipeline_register(base_stream_pipeline, base_raw_write_el, "base_raw");

//const char *link_tag_base[4] = {"base_file", "base_mp3", "base_filter", "base_raw"};
//audio_pipeline_link(base_stream_pipeline, &link_tag_base[0], 4);

const char *link_tag_base[3] = {"base_file", "base_mp3", "base_raw"};
audio_pipeline_link(base_stream_pipeline, &link_tag_base[0], 3);

ringbuf_handle_t rb_base = audio_element_get_input_ringbuf(base_raw_write_el);
downmix_set_input_rb(downmixer, rb_base, 0);
audio_pipeline_set_listener(base_stream_pipeline, evt);

audio_pipeline_handle_t newcome_stream_pipeline = audio_pipeline_init(&pipeline_cfg);
mem_assert(newcome_stream_pipeline);
audio_pipeline_register(newcome_stream_pipeline, newcome_fatfs_reader_el, "newcome_file");
audio_pipeline_register(newcome_stream_pipeline, newcome_mp3_decoder_el, "newcome_mp3");
//audio_pipeline_register(newcome_stream_pipeline, newcome_rsp_filter_el, "newcome_filter");
audio_pipeline_register(newcome_stream_pipeline, newcome_raw_write_el, "newcome_raw");

//const char *link_tag_newcome[4] = {"newcome_file", "newcome_mp3", "newcome_filter", "newcome_raw"};
//audio_pipeline_link(newcome_stream_pipeline, &link_tag_newcome[0], 4);

const char *link_tag_newcome[3] = {"newcome_file", "newcome_mp3", "newcome_raw"};
audio_pipeline_link(newcome_stream_pipeline, &link_tag_newcome[0], 3);

ringbuf_handle_t rb_newcome = audio_element_get_input_ringbuf(newcome_raw_write_el);
downmix_set_input_rb(downmixer, rb_newcome, 1);
audio_pipeline_set_listener(newcome_stream_pipeline, evt);

ESP_LOGI(TAG, "[5.1] Listening event from peripherals");
audio_pipeline_set_listener(pipeline_mix, evt);
audio_event_iface_set_listener(esp_periph_set_get_event_iface(set), evt);
downmix_set_output_type(downmixer, PLAY_STATUS);

audio_pipeline_run(base_stream_pipeline);
audio_pipeline_run(pipeline_mix);
downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_BYPASS);
ESP_LOGI(TAG, "[6.0] Base stream pipeline running");

i2s_stream_set_clk(i2s_writer, SAMPLERATE, 16, PLAY_STATUS);
audio_hal_set_volume(board_handle->audio_hal, 40); 
downmix_set_input_rb_timeout(downmixer, 50, INDEX_BASE_STREAM);
while (1) {
    audio_event_iface_msg_t msg;
    esp_err_t ret = audio_event_iface_listen(evt, &msg, portMAX_DELAY);
    if (ret != ESP_OK) {
        ESP_LOGE(TAG, "[ * ] Event interface error : %d", ret);
        continue;
    }

    if (((int)msg.data == get_input_mode_id()) && (msg.cmd == PERIPH_BUTTON_PRESSED)) {

        audio_pipeline_run(newcome_stream_pipeline);
        downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_ON);
        downmix_set_input_rb_timeout(downmixer, 50, INDEX_NEWCOME_STREAM);
        ESP_LOGI(TAG, "TONE running...");
    }

    if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && 
        msg.source == (void *)i2s_writer && 
        msg.cmd == AEL_MSG_CMD_REPORT_STATUS && 
        (((int)msg.data == AEL_STATUS_STATE_STOPPED) || 
        ((int)msg.data == AEL_STATUS_STATE_FINISHED))){

        ESP_LOGW(TAG, "Player finsihed, break loop");
        break;
    }

    if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && 
        msg.source == (void *)newcome_mp3_decoder_el/*newcome_rsp_filter_el*/ && 
        msg.cmd == AEL_MSG_CMD_REPORT_STATUS && 
        (((int) msg.data == AEL_STATUS_STATE_STOPPED) || 
        ((int)msg.data == AEL_STATUS_STATE_FINISHED))){

        downmix_set_work_mode(downmixer, ESP_DOWNMIX_WORK_MODE_SWITCH_OFF);
        downmix_set_input_rb_timeout(downmixer, 0, INDEX_NEWCOME_STREAM);
        audio_pipeline_stop(newcome_stream_pipeline);
        audio_pipeline_wait_for_stop(newcome_stream_pipeline);
        audio_pipeline_terminate(newcome_stream_pipeline);
        audio_pipeline_reset_ringbuffer(newcome_stream_pipeline);
        audio_pipeline_reset_elements(newcome_stream_pipeline);
        ESP_LOGI(TAG, "TONE finsihed");
    }
}

ESP_LOGI(TAG, "[7.0] Stop all pipelines");
/* Stop base stream pipeline, Release resources */
downmix_set_input_rb_timeout(downmixer, 0, INDEX_BASE_STREAM);
audio_pipeline_stop(base_stream_pipeline);
audio_pipeline_wait_for_stop(base_stream_pipeline);
audio_pipeline_terminate(base_stream_pipeline);
audio_pipeline_unregister_more( base_stream_pipeline, 
                                base_fatfs_reader_el,
                                base_mp3_decoder_el, 
                                //base_rsp_filter_el, 
                                base_raw_write_el, NULL);
audio_pipeline_remove_listener(base_stream_pipeline);
audio_pipeline_deinit(base_stream_pipeline);
audio_element_deinit(base_fatfs_reader_el);
audio_element_deinit(base_mp3_decoder_el);
//audio_element_deinit(base_rsp_filter_el);
audio_element_deinit(base_raw_write_el);

/* Stop newcome stream pipeline, Release resources */
audio_pipeline_stop(newcome_stream_pipeline);
audio_pipeline_wait_for_stop(newcome_stream_pipeline);
audio_pipeline_terminate(newcome_stream_pipeline);
audio_pipeline_unregister_more( newcome_stream_pipeline, 
                                newcome_fatfs_reader_el,
                                newcome_mp3_decoder_el, 
                                //newcome_rsp_filter_el, 
                                newcome_raw_write_el, NULL);
audio_pipeline_remove_listener(newcome_stream_pipeline);
audio_pipeline_deinit(newcome_stream_pipeline);
audio_element_deinit(newcome_fatfs_reader_el);
audio_element_deinit(newcome_mp3_decoder_el);
//audio_element_deinit(newcome_rsp_filter_el);
audio_element_deinit(newcome_raw_write_el);

/* Stop mixer stream pipeline, Release resources */
audio_pipeline_stop(pipeline_mix);
audio_pipeline_wait_for_stop(pipeline_mix);
audio_pipeline_terminate(pipeline_mix);
audio_pipeline_unregister_more(pipeline_mix, downmixer, i2s_writer, NULL);
audio_pipeline_remove_listener(pipeline_mix);

/* Stop all peripherals before removing the listener */
esp_periph_set_stop_all(set);
audio_event_iface_remove_listener(esp_periph_set_get_event_iface(set), evt);

/* Make sure audio_pipeline_remove_listener & audio_event_iface_remove_listener are called before destroying event_iface */
audio_event_iface_destroy(evt);

/* Release resources */
audio_pipeline_deinit(pipeline_mix);
audio_element_deinit(downmixer);
audio_element_deinit(i2s_writer);
esp_periph_set_destroy(set);

} `

0x0fe commented 11 months ago

@majingjing123 Yes, it works now, thank you very much for your help debugging the issue. I guess the example should be updated to reflect these corrections.

Jack-Agilian commented 6 months ago

I'm having the similar issue, with new_come_stream alone. My solution was to increase the 'max_sample' in 'downmix_cfg'. The root cause is that stream processing on a node is too slow.