Support for new I2S driver (AUD-4786)

[zafeer-birde]

Is your feature request related to a problem? Please describe. Currently ESP ADF master does not support the new I2S driver per https://github.com/espressif/esp-idf/blob/master/docs/en/api-reference/peripherals/i2s.rst.

Describe the solution you'd like Changes to be made to support the new I2S driver.

Additional context We needed support for ADC one shot which is why we had to use an updated version of ESP IDF(https://github.com/espressif/esp-idf/releases/tag/v5.0.3) which is not the same one currently being used by the ADF master branch.

In the code we get warnings about migrating to the new I2S driver and a crash on boot with error code

[ADC]: CONFLICT! driver_ng is not allowed to be used with the legacy driver.

I have tried to make the changes to the audio boards and audio stream components but is taking a little longer than expected.

Any help or guidance will also be appreciated.

[jason-mao]

@zafeer-birde Yes, we are aware of this problem. We will provide a somewhat informal fix.

[shootao]

Hi @zafeer-birde Currently, both new i2s APIand legacy I2S API are not allowed to be used simultaneously. If you are using the new i2s API, you can use it in the form of a callback ex:

#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "sdkconfig.h"
#include "audio_element.h"
#include "audio_pipeline.h"
#include "audio_event_iface.h"
#include "audio_common.h"
#include "tts_stream.h"
#include "esp_peripherals.h"
#include "board.h"

bool play_once_flag = true;
static const char *TAG = "PLAY_TTS_EXAMPLE";
static const char *CHINESE_STRINGS = "欢迎使用乐鑫语音开源框架";

#if ((ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0)) && CONFIG_IDF_TARGET_ESP32C3)
#include "driver/i2s_pdm.h"
#define  ENABLE_TTS_EXAMPLE_FOR_ESP32C3
#else
#include "i2s_stream.h"
#endif  

static __attribute__((constructor)) void example_prepare_check(void)
{
#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 1, 0) && CONFIG_IDF_TARGET_ESP32C3
    ESP_LOGE(__func__, "Please update esp-idf version great than or equal to V5.1");
    abort();
#endif
}

#if defined ENABLE_TTS_EXAMPLE_FOR_ESP32C3
static int tts_stream_write_callback_fn (audio_element_handle_t self, char *buffer, int len, TickType_t ticks_to_wait, void *context)
{
    i2s_chan_handle_t tx_ch = (i2s_chan_handle_t) context;
    size_t w_bytes = 0;
    i2s_channel_write(tx_ch, buffer, len, &w_bytes, ticks_to_wait);

    return w_bytes;
}

static i2s_chan_handle_t i2s_pdm_peri_init()
{
    i2s_chan_handle_t tx_chan;
    i2s_chan_config_t tx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_AUTO, I2S_ROLE_MASTER);
    tx_chan_cfg.auto_clear = true;
    ESP_ERROR_CHECK(i2s_new_channel(&tx_chan_cfg, &tx_chan, NULL));

    i2s_pdm_tx_config_t pdm_tx_cfg = {
        .clk_cfg = I2S_PDM_TX_CLK_DEFAULT_CONFIG(16000),
        .slot_cfg = I2S_PDM_TX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO),
        .gpio_cfg = {
            .clk = -1,
            .dout = GPIO_NUM_3,
            .invert_flags = {
                .clk_inv = false,
            },
        },
    };
    ESP_ERROR_CHECK(i2s_channel_init_pdm_tx_mode(tx_chan, &pdm_tx_cfg));
    ESP_ERROR_CHECK(i2s_channel_enable(tx_chan));
    return tx_chan;
}
#endif // ENABLE_TTS_EXAMPLE_FOR_ESP32C3

void app_main(void)
{
    audio_pipeline_handle_t pipeline;
    audio_element_handle_t tts_stream_reader;

    esp_log_level_set("*", ESP_LOG_WARN);
    esp_log_level_set(TAG, ESP_LOG_INFO);

    ESP_LOGI(TAG, "[1.0] Init Peripheral Set");
    esp_periph_config_t periph_cfg = DEFAULT_ESP_PERIPH_SET_CONFIG();
    esp_periph_set_handle_t set = esp_periph_set_init(&periph_cfg);

    ESP_LOGI(TAG, "[2.0] Start codec chip");
    audio_board_handle_t board_handle = audio_board_init();
    audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_DECODE, AUDIO_HAL_CTRL_START);

    ESP_LOGI(TAG, "[3.0] Create audio pipeline for playback");
    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
    pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(pipeline);

    ESP_LOGI(TAG, "[3.1] Create tts stream to read data from chinese strings");
    tts_stream_cfg_t tts_cfg = TTS_STREAM_CFG_DEFAULT();
    tts_cfg.type = AUDIO_STREAM_READER;
    tts_stream_reader = tts_stream_init(&tts_cfg);

    ESP_LOGI(TAG, "[3.2] Register tts elements to audio pipeline");
    audio_pipeline_register(pipeline, tts_stream_reader, "tts");

#if defined ENABLE_TTS_EXAMPLE_FOR_ESP32C3
    ESP_LOGI(TAG, "[3.3] Init i2s pdm to write data to codec chip");
    i2s_chan_handle_t tx_ch = NULL;
    tx_ch = i2s_pdm_peri_init();
    const char *link_tag[1] = {"tts"};
    ESP_LOGI(TAG, "[3.4] Link it together [strings]-->tts_stream-->i2s write callback-->[codec_chip]");
    audio_pipeline_link(pipeline, &link_tag[0], 1);
    ESP_LOGI(TAG, "[3.5] Set tts stream's write callback function");
    audio_element_set_write_cb(tts_stream_reader, tts_stream_write_callback_fn, (void *)tx_ch);
#else
    ESP_LOGI(TAG, "[3.3] Create i2s stream to write data to codec chip");
    audio_element_handle_t i2s_stream_writer;
    i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();
    i2s_cfg.type = AUDIO_STREAM_WRITER;
    i2s_cfg.i2s_config.sample_rate = 16000;
    i2s_cfg.i2s_config.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT;
    i2s_stream_writer = i2s_stream_init(&i2s_cfg);

    ESP_LOGI(TAG, "[3.4] Register i2s stream elements to audio pipeline");
    audio_pipeline_register(pipeline, i2s_stream_writer, "i2s");
    ESP_LOGI(TAG, "[3.5] Link it together [strings]-->tts_stream-->i2s_stream-->[codec_chip]");
    const char *link_tag[2] = {"tts", "i2s"};
    audio_pipeline_link(pipeline, &link_tag[0], 2);
#endif // ENABLE_TTS_EXAMPLE_FOR_ESP32C3

    ESP_LOGI(TAG, "[3.6] Set up  uri (tts as tts_stream, and directly output is i2s)");
    tts_stream_set_strings(tts_stream_reader, CHINESE_STRINGS);

    ESP_LOGI(TAG, "[4.0] Set up  event listener");
    audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
    audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg);

    ESP_LOGI(TAG, "[4.1] Listening event from all elements of pipeline");
    audio_pipeline_set_listener(pipeline, evt);

    ESP_LOGI(TAG, "[4.2] Listening event from peripherals");
    audio_event_iface_set_listener(esp_periph_set_get_event_iface(set), evt);

    ESP_LOGI(TAG, "[5.0] Start audio_pipeline");
    audio_pipeline_run(pipeline);

    ESP_LOGI(TAG, "[6.0] Listen for all pipeline events");
}

[zafeer-birde]

Hi @shootao ,

Thank you for providing an example on integrating the new i2s. I have adapted it for a media playback from the sd card. However there is still the same warning coming from this file board_pins_config.h. This gets added in from the board.h file. Anyway we can resolve this ?

I still haven't tested my code but awaiting a feedback on the warning still being generated due to the above.

Also to confirm some changes below

1. We need to play 2 channels with 16bit and 48000Hz . Are these settings correct ? Also what is the GPIO_NUM_3 ?? For the LyraT 4.3 and ESP32S3 Korvo2 Duo what should this value be ?

static i2s_chan_handle_t i2s_pdm_peri_init()
    {
        i2s_chan_handle_t tx_chan;
        i2s_chan_config_t tx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_AUTO, I2S_ROLE_MASTER);
        tx_chan_cfg.auto_clear = true;
        ESP_ERROR_CHECK(i2s_new_channel(&tx_chan_cfg, &tx_chan, NULL));

        i2s_pdm_tx_config_t pdm_tx_cfg = {
            .clk_cfg = I2S_PDM_TX_CLK_DEFAULT_CONFIG(48000),
            .slot_cfg = I2S_PDM_TX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO),
            .gpio_cfg = {
                .clk = -1,
                .dout = GPIO_NUM_3,
                .invert_flags = {
                    .clk_inv = false,
                },
            },
        };
        ESP_ERROR_CHECK(i2s_channel_init_pdm_tx_mode(tx_chan, &pdm_tx_cfg));
        ESP_ERROR_CHECK(i2s_channel_enable(tx_chan));
        return tx_chan;
    }

2. In certain cases we might need to set the clk depending on the music being played at run time

   eg As per old code below

   i2s_stream_set_clk(i2s_stream_writer_el, music_info.sample_rates, music_info.bits, music_info.channels);

   How do we do this for the new i2s ?

3. Just confirming from your code above that the i2s is added as a cb so no need to register and link per the new i2s update?

4. We are on ESP IDF v 5.0.3 which also seems to support the new i2s driver. Do we need to shift to 5.1 explicitly ?

[zafeer-birde]

Hi @jason-mao and @shootao , Do we have any update on the queries raised ?

[shootao]

Hi @zafeer-birde Pls using the patch in the attachment 0001-i2s_stream-support-idfv5.X-i2s-API.patch

[Zafeer]

Hi @shootao , Thanks for sharing the patch.

I did try to integrate it with the code but I faced some errors while building. Can you confirm if the above patch in combination with the code you have shared is supposed to be attempted ?

Also I'm not sure if the patch answers all my questions. Can you confirm these as well once ?

[shootao]

Hi @zafeer-birde This patch base on follow commit

commit 58a6c3728f89e453daa391448b0799f38a54bdfc (origin/master, origin/HEAD, master)
Merge: 47ce3037 070ed11f
Author: Jason-mao <maojianxin@espressif.com>
Date:   Fri Jul 21 22:24:21 2023 +0800

    Merge branch 'feature/support_multiple_media_for_fatfs_stream' into 'master'

    fatfs: Remove fatfs stream's restriction on sdcard usage only

    See merge request adf/esp-adf-internal!1191

[shootao]

Hi @Zafeer Can this patch solve your problem？

[Zafeer]

Hi @shootao , I'm unsure if I performed all patches correctly as suggested.

Can you link all changes to a repo if possible ?

I tried with my understanding but the Audio Pipeline hangs and does not play using the new I2S.

[shootao]

@Zafeer Sorry, This may not be submitted as an official version at the moment. This usage is exactly the same as before. You need to just pay attention when initializing i2s_stream.For example, If you want to use pdm mode to write

    i2s_stream_cfg_t i2s_cfg = I2S_STREAM_TX_PDM_CFG_DEFAULT();
    i2s_cfg.type = AUDIO_STREAM_WRITER;
    i2s_stream_writer = i2s_stream_init(&i2s_cfg);

[strdvxt]

Hi @zafeer-birde Currently, both new i2s APIand legacy I2S API are not allowed to be used simultaneously. If you are using the new i2s API, you can use it in the form of a callback ex:

#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "sdkconfig.h"
#include "audio_element.h"
#include "audio_pipeline.h"
#include "audio_event_iface.h"
#include "audio_common.h"
#include "tts_stream.h"
#include "esp_peripherals.h"
#include "board.h"

bool play_once_flag = true;
static const char *TAG = "PLAY_TTS_EXAMPLE";
static const char *CHINESE_STRINGS = "欢迎使用乐鑫语音开源框架";

#if ((ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0)) && CONFIG_IDF_TARGET_ESP32C3)
#include "driver/i2s_pdm.h"
#define  ENABLE_TTS_EXAMPLE_FOR_ESP32C3
#else
#include "i2s_stream.h"
#endif  

static __attribute__((constructor)) void example_prepare_check(void)
{
#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 1, 0) && CONFIG_IDF_TARGET_ESP32C3
    ESP_LOGE(__func__, "Please update esp-idf version great than or equal to V5.1");
    abort();
#endif
}

#if defined ENABLE_TTS_EXAMPLE_FOR_ESP32C3
static int tts_stream_write_callback_fn (audio_element_handle_t self, char *buffer, int len, TickType_t ticks_to_wait, void *context)
{
    i2s_chan_handle_t tx_ch = (i2s_chan_handle_t) context;
    size_t w_bytes = 0;
    i2s_channel_write(tx_ch, buffer, len, &w_bytes, ticks_to_wait);

    return w_bytes;
}

static i2s_chan_handle_t i2s_pdm_peri_init()
{
    i2s_chan_handle_t tx_chan;
    i2s_chan_config_t tx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_AUTO, I2S_ROLE_MASTER);
    tx_chan_cfg.auto_clear = true;
    ESP_ERROR_CHECK(i2s_new_channel(&tx_chan_cfg, &tx_chan, NULL));

    i2s_pdm_tx_config_t pdm_tx_cfg = {
        .clk_cfg = I2S_PDM_TX_CLK_DEFAULT_CONFIG(16000),
        .slot_cfg = I2S_PDM_TX_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_16BIT, I2S_SLOT_MODE_MONO),
        .gpio_cfg = {
            .clk = -1,
            .dout = GPIO_NUM_3,
            .invert_flags = {
                .clk_inv = false,
            },
        },
    };
    ESP_ERROR_CHECK(i2s_channel_init_pdm_tx_mode(tx_chan, &pdm_tx_cfg));
    ESP_ERROR_CHECK(i2s_channel_enable(tx_chan));
    return tx_chan;
}
#endif // ENABLE_TTS_EXAMPLE_FOR_ESP32C3

void app_main(void)
{
    audio_pipeline_handle_t pipeline;
    audio_element_handle_t tts_stream_reader;

    esp_log_level_set("*", ESP_LOG_WARN);
    esp_log_level_set(TAG, ESP_LOG_INFO);

    ESP_LOGI(TAG, "[1.0] Init Peripheral Set");
    esp_periph_config_t periph_cfg = DEFAULT_ESP_PERIPH_SET_CONFIG();
    esp_periph_set_handle_t set = esp_periph_set_init(&periph_cfg);

    ESP_LOGI(TAG, "[2.0] Start codec chip");
    audio_board_handle_t board_handle = audio_board_init();
    audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_DECODE, AUDIO_HAL_CTRL_START);

    ESP_LOGI(TAG, "[3.0] Create audio pipeline for playback");
    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
    pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(pipeline);

    ESP_LOGI(TAG, "[3.1] Create tts stream to read data from chinese strings");
    tts_stream_cfg_t tts_cfg = TTS_STREAM_CFG_DEFAULT();
    tts_cfg.type = AUDIO_STREAM_READER;
    tts_stream_reader = tts_stream_init(&tts_cfg);

    ESP_LOGI(TAG, "[3.2] Register tts elements to audio pipeline");
    audio_pipeline_register(pipeline, tts_stream_reader, "tts");

#if defined ENABLE_TTS_EXAMPLE_FOR_ESP32C3
    ESP_LOGI(TAG, "[3.3] Init i2s pdm to write data to codec chip");
    i2s_chan_handle_t tx_ch = NULL;
    tx_ch = i2s_pdm_peri_init();
    const char *link_tag[1] = {"tts"};
    ESP_LOGI(TAG, "[3.4] Link it together [strings]-->tts_stream-->i2s write callback-->[codec_chip]");
    audio_pipeline_link(pipeline, &link_tag[0], 1);
    ESP_LOGI(TAG, "[3.5] Set tts stream's write callback function");
    audio_element_set_write_cb(tts_stream_reader, tts_stream_write_callback_fn, (void *)tx_ch);
#else
    ESP_LOGI(TAG, "[3.3] Create i2s stream to write data to codec chip");
    audio_element_handle_t i2s_stream_writer;
    i2s_stream_cfg_t i2s_cfg = I2S_STREAM_CFG_DEFAULT();
    i2s_cfg.type = AUDIO_STREAM_WRITER;
    i2s_cfg.i2s_config.sample_rate = 16000;
    i2s_cfg.i2s_config.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT;
    i2s_stream_writer = i2s_stream_init(&i2s_cfg);

    ESP_LOGI(TAG, "[3.4] Register i2s stream elements to audio pipeline");
    audio_pipeline_register(pipeline, i2s_stream_writer, "i2s");
    ESP_LOGI(TAG, "[3.5] Link it together [strings]-->tts_stream-->i2s_stream-->[codec_chip]");
    const char *link_tag[2] = {"tts", "i2s"};
    audio_pipeline_link(pipeline, &link_tag[0], 2);
#endif // ENABLE_TTS_EXAMPLE_FOR_ESP32C3

    ESP_LOGI(TAG, "[3.6] Set up  uri (tts as tts_stream, and directly output is i2s)");
    tts_stream_set_strings(tts_stream_reader, CHINESE_STRINGS);

    ESP_LOGI(TAG, "[4.0] Set up  event listener");
    audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
    audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg);

    ESP_LOGI(TAG, "[4.1] Listening event from all elements of pipeline");
    audio_pipeline_set_listener(pipeline, evt);

    ESP_LOGI(TAG, "[4.2] Listening event from peripherals");
    audio_event_iface_set_listener(esp_periph_set_get_event_iface(set), evt);

    ESP_LOGI(TAG, "[5.0] Start audio_pipeline");
    audio_pipeline_run(pipeline);

    ESP_LOGI(TAG, "[6.0] Listen for all pipeline events");
}

hey do you really has solved the problem? which line is called "callback" and where could I put the line?

[jason-mao]

It's add in https://github.com/espressif/esp-adf/blob/master/components/audio_stream/i2s_stream_idf5.c

[jason-mao]

@strdvxt I don't see the details of your question. You can create a new issue for it