Reuse/Idle Pipeline When it is finished (AUD-6795)

[Haerteleric]

Checklist

• Checked the issue tracker for similar issues to ensure this is not a duplicate.
• Provided a clear description of your suggestion.
• Included any relevant context or examples.

Issue or Suggestion Description

Description:
Hi, I’m using ESP-GMF to develop an interactive game. My goal is to stitch together a story from multiple MP3 files.

The challenge is that each file has a different runtime, so they may need to start/stop while another file is still playing. I also need to reuse pipelines to play different files once the current one is done (e.g., in a conversation).

Here’s a diagram illustrating my approach:

My current GMF architecture:

• Each track and the mixer have their own GMF-Task assigned.
• They are connected via a ring buffer at the moment.

Problems encountered:

1. If I start the GMF-Task and one of the track pipelines is not started or has no input URI, there is no sound output at all.
2. If one track pipeline finishes and I try to restart it with a different URI, I get a Task Timeout when attempting to restart the pipeline.

Question:

What is the correct way to:

• Keep pipelines functional even if some don’t have a URI yet?
• Properly restart a finished pipeline with a new URI without hitting a timeout?

This is my current Soucecode (Work in Progress, Skratch State)

#include "game_audio.h"
#include "utils/pinMap.h"
#include "sdkconfig.h"

#include "esp_gmf_io_codec_dev.h"
#include "esp_gmf_port.h"
#include "esp_gmf_pool.h"
#include "esp_gmf_pool.h"
#include "esp_gmf_err.h"


const char *TAG = "GAME_AUDIO";

#ifndef AUDIO_MANAGER_DEBUG_INFO
    #define AUDIO_MANAGER_DEBUG_INFO(...)
#endif

#ifndef AUDIO_MANAGER_DEBUG_ERROR
    #define AUDIO_MANAGER_DEBUG_ERROR(...)
#endif

#define GAME_AUDIO_EXPECT(func, expect, retVar) \
    do { \
        retVar = func; \
        if (retVar != expect) { \
            AUDIO_MANAGER_DEBUG_ERROR("'%s' failed at %s:%d", #func, __FILE__, __LINE__); \
            goto _exit; \
        } \
    } while (0)

//*-----------------------------GMF Audio IO Init Functions------------------------*/
#include "esp_gmf_io.h"
#include "esp_gmf_io_file.h"
#include "esp_gmf_io_codec_dev.h"
static esp_gmf_err_t initGmfIo_fileReader(esp_gmf_pool_handle_t pool)
{
    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;
    file_io_cfg_t fs_cfg = FILE_IO_CFG_DEFAULT();
    esp_gmf_io_handle_t hd = NULL;
    fs_cfg.dir = ESP_GMF_IO_DIR_READER;
    ret = esp_gmf_io_file_init(&fs_cfg, &hd);
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init file io");
    ret = esp_gmf_pool_register_io(pool, hd, "fileReader");
    ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_io_deinit(hd); return ret;}, "Failed to register file io");
    return ret;
}

static esp_gmf_err_t initGmfIo_codecWriter(esp_gmf_pool_handle_t pool)
{
    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);

    esp_gmf_err_t ret = ESP_GMF_ERR_OK;
    esp_gmf_io_handle_t dev = NULL;
    codec_dev_io_cfg_t tx_codec_dev_cfg = ESP_GMF_IO_CODEC_DEV_CFG_DEFAULT();
    tx_codec_dev_cfg.dir = ESP_GMF_IO_DIR_WRITER;
    tx_codec_dev_cfg.dev = NULL;
    ret = esp_gmf_io_codec_dev_init(&tx_codec_dev_cfg, &dev);
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init codec dev io");
    ret = esp_gmf_pool_register_io(pool, dev, "codecWriter");
    ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_io_deinit(dev); return ret;}, "Failed to register codec dev io");
    return ret;
}

//*--------------------------GMF Audio Decoder Init Functions----------------------*/
#include "esp_audio_dec_default.h"
#include "esp_audio_enc_default.h"
#include "esp_audio_simple_dec_default.h"
#include "esp_gmf_audio_dec.h"

static esp_gmf_err_t initGmfElement_mp3Decoder(esp_gmf_pool_handle_t pool)
{
    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;

    ret = (esp_gmf_err_t) esp_mp3_dec_register();
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to register mp3 decoder");
    
    esp_audio_simple_dec_cfg_t dec_cfg = DEFAULT_ESP_GMF_AUDIO_DEC_CONFIG();    
    esp_gmf_element_handle_t hd = NULL;
    dec_cfg.dec_type = ESP_AUDIO_SIMPLE_DEC_TYPE_MP3;

    ret = esp_gmf_audio_dec_init(&dec_cfg, &hd);
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio dec");
    ret = esp_gmf_pool_register_element(pool, hd, "mp3Decoder");
    ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_element_deinit(hd); return ret;}, "Failed to register element in pool");
    return ret;
}

//*--------------------------GMF Audio Elements Init Functions---------------------*/
#include "esp_gmf_mixer.h"
#include "esp_gmf_rate_cvt.h"
#include "esp_gmf_bit_cvt.h"
#include "esp_gmf_ch_cvt.h"
static esp_gmf_err_t initGmfElement_mixer(esp_gmf_pool_handle_t pool, uint8_t numAudioTracks)
{
    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;
    esp_gmf_element_handle_t hd = NULL;

    // Configure Track Info for Mixer
    esp_ae_mixer_info_t *mixerInputConfig = (esp_ae_mixer_info_t *)malloc(sizeof(esp_ae_mixer_info_t) * numAudioTracks);
    if (mixerInputConfig == NULL) {
        return ESP_GMF_ERR_MEMORY_LACK;
    }
    for(int i = 0; i < numAudioTracks; i++) {
        mixerInputConfig[i].weight1 = 1.0f; // Default weight for each track
        mixerInputConfig[i].weight2 = 1.0f; // Default weight for each track
        mixerInputConfig[i].transit_time = 100; // Default transition time of 100 ms
    }

    // Configure Mixer
    esp_ae_mixer_cfg_t mixer_cfg = 
    {
        .channel = 1,
        .sample_rate = 44100,
        .bits_per_sample = 16,
        .src_num = numAudioTracks,
        .src_info = mixerInputConfig
    };

    // Initialize Mixer Element
    ret = esp_gmf_mixer_init(&mixer_cfg, &hd);
    free(mixerInputConfig); // Free the mixer input config after use
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio mixer");

    // Register Mixer Element
    ret = esp_gmf_pool_register_element(pool, hd, "mixer");
    ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_element_deinit(hd); return ret;}, "Failed to register element in pool");
    return ret;
}

static esp_gmf_err_t initGmfElement_sampleRateConverter(esp_gmf_pool_handle_t pool)
{
    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;
    esp_gmf_element_handle_t hd = NULL;
    esp_ae_rate_cvt_cfg_t es_rate_cvt_cfg = DEFAULT_ESP_GMF_RATE_CVT_CONFIG();

    es_rate_cvt_cfg.dest_rate = 44100;
    es_rate_cvt_cfg.complexity = 2; // CPU vs Quality tradeoff (1=fast/low, 3=slow/high)
    es_rate_cvt_cfg.perf_type = ESP_AE_RATE_CVT_PERF_TYPE_SPEED;

    ret = esp_gmf_rate_cvt_init(&es_rate_cvt_cfg, &hd);
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init sampleRateConverter");
    ret = esp_gmf_pool_register_element(pool, hd, "sampleRateConverter");
    ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_element_deinit(hd); return ret;}, "Failed to register sampleRateConverter element in pool");
    return ret;
}

static esp_gmf_err_t initGmfElement_bitDepthConverter(esp_gmf_pool_handle_t pool)
{
    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;
    esp_gmf_element_handle_t hd = NULL;
    esp_ae_bit_cvt_cfg_t es_bit_cvt_cfg = DEFAULT_ESP_GMF_BIT_CVT_CONFIG();

    es_bit_cvt_cfg.dest_bits = 16;

    ret = esp_gmf_bit_cvt_init(&es_bit_cvt_cfg, &hd);
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init audio bit depth converter");
    ret = esp_gmf_pool_register_element(pool, hd, "bitDepthConverter");
    ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_element_deinit(hd); return ret;}, "Failed to register audio bit depth converter element in pool");
    return ret;
}

static esp_gmf_err_t initGmfElement_channelConverter(esp_gmf_pool_handle_t pool)
{
    //There are 2 chCvt needed:
    // 1. Stereo to Mono for the input tracks (always mono output)
    // 2. Mono to Stereo for the final output to the DAC (always stereo output

    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;
    esp_gmf_element_handle_t handleIn = NULL;

    // 1. Stereo to Mono for the input tracks (always mono output)
    esp_ae_ch_cvt_cfg_t es_ch_cvt_cfg = DEFAULT_ESP_GMF_CH_CVT_CONFIG();
    es_ch_cvt_cfg.dest_ch = 1; // Always mono

    ret = esp_gmf_ch_cvt_init(&es_ch_cvt_cfg, &handleIn);
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init channelConverter_in");
    ret = esp_gmf_pool_register_element(pool, handleIn, "channelConverter_in");
    ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_element_deinit(handleIn); return ret;}, "Failed to register channelConverter_in element in pool");

    // 2. Mono to Stereo for the final output to the DAC (always stereo output)
    esp_gmf_element_handle_t handleOut = NULL; // Reset handle for new element
    es_ch_cvt_cfg.dest_ch = 2; // Always stereo

    ret = esp_gmf_ch_cvt_init(&es_ch_cvt_cfg, &handleOut);
    ESP_GMF_RET_ON_ERROR(TAG, ret, return ret, "Failed to init channelConverter_out");
    ret = esp_gmf_pool_register_element(pool, handleOut, "channelConverter_out");
ESP_GMF_RET_ON_ERROR(TAG, ret, {esp_gmf_element_deinit(handleOut); return ret;}, "Failed to register channelConverter_out element in pool");

    return ret;
}

//*-------------------------GMF Audio Track Pipeline Functions-----------------------*//
#include "esp_gmf_pipeline.h"
#include "esp_gmf_task.h"
#include "esp_gmf_data_bus.h"
#include "esp_gmf_new_databus.h"
#include "freertos/FreeRTOS.h"
#include "freertos/event_groups.h"

typedef struct game_audio_track_s{
    esp_gmf_task_handle_t task;
    esp_gmf_pipeline_handle_t pipeline;
    esp_gmf_db_handle_t db;
} game_audio_track_t;

esp_gmf_err_t trackEvent_callback(esp_gmf_event_pkt_t *pkt, void *ctx)
{
    return ESP_GMF_ERR_OK;
}

static esp_gmf_err_t initGmfPipeline_track(esp_gmf_pool_handle_t pool, game_audio_track_t *track, int trackIndex)
{
        ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
        esp_gmf_err_t ret = ESP_GMF_ERR_OK;

        // Create unique task name for each track
        char trackTaskName[8] = "track_0"; // Base name for track tasks
        trackTaskName[6] = '0' + trackIndex; // Set the track number in the task name

        //Create Ring Buffer for each track
        if(esp_gmf_db_new_ringbuf(4, 1024, &track->db) < 0) {
            return ESP_GMF_ERR_MEMORY_LACK; // GMF Data Bus creation failed
        }

        // Initialize each track's pipeline
        const char *elements[] = {"mp3Decoder", "channelConverter_in", "bitDepthConverter", "sampleRateConverter"};
        const uint8_t num_elements = sizeof(elements) / sizeof(elements[0]);
        if(esp_gmf_pool_new_pipeline(pool, "fileReader", elements, num_elements, NULL, &track->pipeline) != ESP_GMF_ERR_OK) {
            AUDIO_MANAGER_DEBUG_ERROR("GMF Pool new pipeline failed for track %d", trackIndex);
            ret = ESP_GMF_ERR_FAIL;
            goto exit; // GMF Pool new pipeline failed
        }

        esp_gmf_task_cfg_t task_cfg = {
            .thread = {
                .stack = 4096, // Stack size for the track task
                .prio = 6, // Priority of the track task
                .core = 0, // Run on core 0
                .stack_in_ext = 1 // Stack is in external memory (PSRAM)
            },
            .name = trackTaskName, // Name of the track task
            .ctx = NULL, // No user context needed
            .cb = NULL // No callback function needed
        };

        if(esp_gmf_task_init(&task_cfg, &track->task) != ESP_GMF_ERR_OK) {
            AUDIO_MANAGER_DEBUG_ERROR("GMF Task initialization failed for track %d", trackIndex);
            ret = ESP_GMF_ERR_FAIL;
            goto exit; // GMF Task initialization failed
        }

        if(esp_gmf_pipeline_bind_task(track->pipeline, track->task) != ESP_GMF_ERR_OK) {
            AUDIO_MANAGER_DEBUG_ERROR("GMF Pipeline bind task failed for track %d", trackIndex);
            ret = ESP_GMF_ERR_FAIL;
            goto exit; // GMF Pipeline bind task failed
        }

        if(esp_gmf_pipeline_loading_jobs(track->pipeline) != ESP_GMF_ERR_OK) {
            AUDIO_MANAGER_DEBUG_ERROR("GMF Pipeline loading jobs failed for track %d", trackIndex);
            ret = ESP_GMF_ERR_FAIL;
            goto exit; // GMF Pipeline loading jobs failed
        }


exit:
    if(ret != ESP_GMF_ERR_OK) {
        if(track->task) {
            esp_gmf_task_deinit(track->task);
            track->task = NULL;
        }
        if(track->pipeline) {
            esp_gmf_pipeline_stop(track->pipeline);
            track->pipeline = NULL;
        }
        if(track->db) {
            esp_gmf_db_deinit(track->db);
            track->db = NULL;
        }
    }
    return ret;
}

typedef struct game_audio_mixer_s{
    esp_gmf_task_handle_t task;
    esp_gmf_pipeline_handle_t pipeline;
} game_audio_mixer_t;

static esp_gmf_err_t initGmfPipeline_mixer(esp_gmf_pool_handle_t pool, game_audio_mixer_t *mixer, game_audio_track_t *tracks, int numTracks)
{
    ESP_GMF_NULL_CHECK(TAG, pool, return ESP_GMF_ERR_INVALID_ARG);
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;

    // Create Mixer Pipeline
    const char *elements[] = {"mixer", "channelConverter_out"};
    const uint8_t numElements = sizeof(elements) / sizeof(elements[0]);

    if(esp_gmf_pool_new_pipeline(pool, NULL, elements, numElements, "codecWriter", &mixer->pipeline) != ESP_GMF_ERR_OK) {
        AUDIO_MANAGER_DEBUG_ERROR("GMF Pool new pipeline failed on mixer");
        return ESP_GMF_ERR_FAIL; // GMF Pool new pipeline failed
    }

    // Connect each track's output to the mixer input
    for(int i = 0; i < numTracks; i++) {
        if(tracks[i].pipeline == NULL) {
            AUDIO_MANAGER_DEBUG_ERROR("Track %d pipeline is NULL", i);
            ret = ESP_GMF_ERR_FAIL;
            goto exit; // Track pipeline is NULL
        }

        esp_gmf_port_handle_t out_port = 
            NEW_ESP_GMF_PORT_OUT_BYTE(
                esp_gmf_db_acquire_write, 
                esp_gmf_db_release_write, 
                esp_gmf_db_deinit, 
                tracks[i].db,
                4096, 300
            )
        ;
        esp_gmf_port_handle_t in_port = 
            NEW_ESP_GMF_PORT_IN_BYTE(
                esp_gmf_db_acquire_read, 
                esp_gmf_db_release_read, 
                esp_gmf_db_deinit, 
                tracks[i].db,
                4096, 300
            )
        ;

        if(esp_gmf_pipeline_connect_pipe(
            tracks[i].pipeline, "sampleRateConverter", out_port,
            mixer->pipeline, "mixer", in_port) != ESP_GMF_ERR_OK) {
            AUDIO_MANAGER_DEBUG_ERROR("GMF Pipeline connect pipe failed for track %d", i);
            ret = ESP_GMF_ERR_FAIL;
            goto exit; // GMF Pipeline connect pipe failed
        }
    }


    esp_gmf_task_cfg_t task_cfg = {
        .thread = {
            .stack = 4096, // Stack size for the track task
            .prio = 5, // Priority of the track task
            .core = 0, // Run on core 0
            .stack_in_ext = 1 // Stack is in external memory (PSRAM)
        },
        .name = "mixer", // Name of the track task
        .ctx = NULL, // No user context needed
        .cb = NULL // No callback function needed
    };

    if(esp_gmf_task_init(&task_cfg, &mixer->task) != ESP_GMF_ERR_OK) {
        AUDIO_MANAGER_DEBUG_ERROR("GMF Task initialization failed for mixer");
        ret = ESP_GMF_ERR_FAIL;
        goto exit; // GMF Task initialization failed
    }

    if(esp_gmf_pipeline_bind_task(mixer->pipeline, mixer->task) != ESP_GMF_ERR_OK) {
        AUDIO_MANAGER_DEBUG_ERROR("GMF Pipeline bind task failed for mixer");
        ret = ESP_GMF_ERR_FAIL;
        goto exit; // GMF Pipeline bind task failed
    }

    if(esp_gmf_pipeline_loading_jobs(mixer->pipeline) != ESP_GMF_ERR_OK) {
        AUDIO_MANAGER_DEBUG_ERROR("GMF Pipeline loading jobs failed for mixer");
        ret = ESP_GMF_ERR_FAIL;
        goto exit; // GMF Pipeline loading jobs failed
    }
exit:
    if(ret != ESP_GMF_ERR_OK) {
        if(mixer->task) {
            esp_gmf_task_deinit(mixer->task);
            mixer->task = NULL;
        }
        if(mixer->pipeline) {
            esp_gmf_pipeline_stop(mixer->pipeline);
            mixer->pipeline = NULL;
        }
    }
    return ret;
}


//*-------------------------ESP Codec Device Functions-----------------------*//
#include "tlv320dac3100.h"

static esp_codec_dev_handle_t s_codecDev = NULL;
esp_codec_dev_handle_t codecDev_init(void)
{
    if(s_codecDev != NULL) {
        return s_codecDev; // Already initialized
    }
    //Init I2C
    i2c_master_bus_config_t i2c_cfg = {
        .sda_io_num = RFID_I2C_SDA_PIN,
        .scl_io_num = RFID_I2C_SCL_PIN,
        .clk_source = I2C_CLK_SRC_DEFAULT,
        .i2c_port = I2C_NUM_0,
        .flags.enable_internal_pullup = 1,
        .glitch_ignore_cnt = 7,
    };
    i2c_master_bus_handle_t i2cHandle = NULL;
    ESP_ERROR_CHECK(i2c_new_master_bus(&i2c_cfg, &i2cHandle));
    
    tlv320dac3100_codec_cfg_t codec_cfg = {
        .ctrl_if = tlv320dac3100_codec_ctrl_if_new(i2cHandle,i2c_cfg.i2c_port),
        .enableHeadphoneDetection = true,
        .monitoringPollIntervalMs = 300, //ms

        
        .pll = {
            .enable = true,
            .clkIn = 0b01, // BCLK

            //BCLK = Fin = 44.1kHz * 2 * 16 = 1.4112MHz

            .p = 1,
            .r = 8,
            .j = 8,
            .d = 0,

            //PLL Fout = (Fin * R * (J + D/16384)) / P) = (1.4112MHz * 8 * (8 + 0/16384)) / 1) = 90.3168MHz
            //(I)   512 KHz <= Fin/P <= 20 MHz
            //(II)  80MHz <= Fout <= 110MHz 
            //(III) 4 <= R*J <= 259
            //(see condition (7) TLV320DAC3100 datasheet (page 55))
        },

        .clk =
        {
            .clkIn = 0b11, // PLL_CLK
            .nDacDivider = 4, // DAC_CLK = 90.3168MHz / 4 = 22.5792MHz
            .mDacDivider = 4, // DAC_MOD_CLK = 22.5792MHz / 4 = 5.6448MHz
            .dOsr = 128, // Fs = 5.6448MHz / 128 = 44.1kHz

            // (I)      DAC_CLK <=  48 MHz
            // (II)     DAC_MOD_CLK <= 6.758 MHz
            // (III)    Fs == 44.1kHz
            // (IV)     2.8 MHz < (DOSR * DAC_fS) < 6.2 MHz
            // (V)      (MDAC * DOSR / 32) ≥ RC_FILTER (max 12)
        },

        .timer = 
        {
            .useMClk = false   
        },
    };

    esp_codec_dev_cfg_t codec_dev_cfg = {
        .dev_type = ESP_CODEC_DEV_TYPE_OUT,
    };
    codec_dev_cfg.data_if = tlv320dac3100_codec_data_if_new(
        AUDIO_AMP_I2S_MCLK_PIN, 
        AUDIO_AMP_I2S_BCLK_PIN, 
        AUDIO_AMP_I2S_WS_PIN,
        AUDIO_AMP_I2S_DOUT_PIN
    );
    codec_dev_cfg.codec_if = tlv320dac3100_codec_if_new(&codec_cfg);

    s_codecDev = esp_codec_dev_new(&codec_dev_cfg);
    return s_codecDev;
}



static esp_gmf_pool_handle_t s_pool = NULL;
static game_audio_mixer_t s_mixer = {0};
static game_audio_track_t *s_tracks = NULL;
static int s_numAudioTracks = 0;
static bool s_mixerRunning = false;

int game_audio_init(int numAudioTracks)
{
    esp_gmf_err_t ret = ESP_GMF_ERR_OK;

    esp_gmf_pool_init(&s_pool);

    GAME_AUDIO_EXPECT(initGmfElement_bitDepthConverter(s_pool), ESP_GMF_ERR_OK, ret);
    GAME_AUDIO_EXPECT(initGmfElement_channelConverter(s_pool), ESP_GMF_ERR_OK, ret);
    GAME_AUDIO_EXPECT(initGmfElement_sampleRateConverter(s_pool), ESP_GMF_ERR_OK, ret);
    GAME_AUDIO_EXPECT(initGmfElement_mp3Decoder(s_pool), ESP_GMF_ERR_OK, ret);
    GAME_AUDIO_EXPECT(initGmfElement_mixer(s_pool, numAudioTracks), ESP_GMF_ERR_OK, ret);
    GAME_AUDIO_EXPECT(initGmfIo_fileReader(s_pool), ESP_GMF_ERR_OK, ret);
    GAME_AUDIO_EXPECT(initGmfIo_codecWriter(s_pool), ESP_GMF_ERR_OK, ret);

    s_tracks = (game_audio_track_t *)malloc(sizeof(game_audio_track_t) * numAudioTracks);
    if(s_tracks == NULL) {
        ret = ESP_GMF_ERR_MEMORY_LACK;
        goto _exit;
    }
    s_numAudioTracks = numAudioTracks;

    for(int i = 0; i < numAudioTracks; i++) {
        memset(&s_tracks[i], 0, sizeof(game_audio_track_t));
        GAME_AUDIO_EXPECT(initGmfPipeline_track(s_pool, &s_tracks[i], i), ESP_GMF_ERR_OK, ret);
    }

    GAME_AUDIO_EXPECT(initGmfPipeline_mixer(s_pool, &s_mixer, s_tracks, numAudioTracks), ESP_GMF_ERR_OK, ret);


    codecDev_init();
    esp_codec_dev_sample_info_t fs = {
        .bits_per_sample = 16,
        .sample_rate = 44100,
        .channel = 2,
    };
    esp_codec_dev_open(s_codecDev, &fs);
    esp_codec_dev_set_out_vol(s_codecDev, 100);
    GAME_AUDIO_EXPECT(esp_gmf_io_codec_dev_set_dev(ESP_GMF_PIPELINE_GET_OUT_INSTANCE(s_mixer.pipeline), s_codecDev), ESP_GMF_ERR_OK, ret);

    // esp_gmf_pipeline_run(s_mixer.pipeline);
    // for (size_t i = 0; i < numAudioTracks; i++)
    // {
    //     esp_gmf_pipeline_run(s_tracks[i].pipeline);
    // }
    

    return ESP_GMF_ERR_OK;

_exit:
    if(ret != ESP_GMF_ERR_OK) {
        game_audio_deinit();
    }
    return ret;
}

int game_audio_deinit()
{
    if(s_tracks) {
        for(int i = 0; i < s_numAudioTracks; i++) {
            if(s_tracks[i].task) {
                esp_gmf_task_deinit(s_tracks[i].task);
                s_tracks[i].task = NULL;
            }
            if(s_tracks[i].pipeline) {
                esp_gmf_pipeline_stop(s_tracks[i].pipeline);
                s_tracks[i].pipeline = NULL;
            }
            if(s_tracks[i].db) {
                esp_gmf_db_deinit(s_tracks[i].db);
                s_tracks[i].db = NULL;
            }
        }
        free(s_tracks);
        s_tracks = NULL;
    }
    if(s_mixer.task) {
        esp_gmf_task_deinit(s_mixer.task);
        s_mixer.task = NULL;
    }
    if(s_mixer.pipeline) {
        esp_gmf_pipeline_stop(s_mixer.pipeline);
        esp_gmf_pipeline_destroy(s_mixer.pipeline);
        s_mixer.pipeline = NULL;
    }

    esp_gmf_pool_deinit(s_pool);


    return ESP_GMF_ERR_OK;
}

int game_audio_playTrack(int trackIndex, const char *filePath)
{
    ESP_LOGI(TAG, "Playing track %d: %s", trackIndex, filePath);
    esp_gmf_pipeline_reset(s_tracks[trackIndex].pipeline);
    esp_gmf_pipeline_set_in_uri(s_tracks[trackIndex].pipeline, filePath);
    esp_gmf_pipeline_run(s_tracks[trackIndex].pipeline);

    if(!s_mixerRunning) {
        esp_gmf_pipeline_run(s_mixer.pipeline);
        s_mixerRunning = true;
    }

    return ESP_GMF_ERR_OK;
}

used like this:

    game_audio_init(2);
    game_audio_playTrack(1, "/sdcard/B.mp3"); //(B.mp3 is 29.8 secs)
    game_audio_playTrack(0, "/sdcard/A.mp3"); //(A.mp3 is 14.5 secs)
    vTaskDelay(pdMS_TO_TICKS(15000)); // Wait for a 15 second (A.mp3 is 14.5 secs)
    game_audio_playTrack(0, "/sdcard/A.mp3"); //Restart just to test
    vTaskDelay(pdMS_TO_TICKS(15000)); // Let Audio Play

[TempoTian]

If restart pipeline you need reset it firstly call esp_gmf_pipeline_reset, see esp_capture code https://github.com/espressif/esp-gmf/blob/main/packages/esp_capture/impl/capture_gmf_path/src/gmf_capture_path_mngr.c#L325 which reuse pipeline.

For mixer issue we will release esp_audio_render module soon which support multiple stream playback simutanously, I think it can meet your request.
Currently mixer element works well when all input port have data, if one port no data, you need set its read timeout to 0 to temp fix the issue.

[Haerteleric]

I updated the port initialization to the following:

esp_gmf_port_handle_t out_port = 
    NEW_ESP_GMF_PORT_OUT_BYTE(
        esp_gmf_db_acquire_write, 
        esp_gmf_db_release_write, 
        esp_gmf_db_deinit, 
        tracks[i].db,
        4096, 0
    );

esp_gmf_port_handle_t in_port = 
    NEW_ESP_GMF_PORT_IN_BYTE(
        esp_gmf_db_acquire_read, 
        esp_gmf_db_release_read, 
        esp_gmf_db_deinit, 
        tracks[i].db,
        4096, 0
    );

However, when I play only one track, the playback speed becomes extremely slow — about 0.33× normal speed.

Interestingly, if I start two tracks and one finishes, the remaining track plays to the end at normal speed.

---

Regarding the esp_gmf_pipeline_reset suggestion:
It doesn’t seem to work with the assigned GMF task. When I run:

ESP_LOGI(TAG, "Playing track %d: %s", trackIndex, filePath);

esp_gmf_event_state_t taskState;
esp_gmf_task_get_state(s_tracks[trackIndex].task, &taskState);
printf("Track %d state: %s\n", trackIndex, esp_gmf_event_get_state_str(taskState));

if (taskState != ESP_GMF_EVENT_STATE_INITIALIZED) {
    esp_gmf_pipeline_reset(s_tracks[trackIndex].pipeline);
}

esp_gmf_pipeline_set_in_uri(s_tracks[trackIndex].pipeline, filePath);
esp_gmf_pipeline_run(s_tracks[trackIndex].pipeline);

I get the following error:

E (20460) ESP_GMF_TASK: Run timeout,[track_1,0x4085f2a0]

This originates from here:

esp-gmf/gmf_core/src/esp_gmf_task.c

Line 475 in 94f7c70

ESP_LOGE(TAG, "Run timeout,[%s,%p]", OBJ_GET_TAG((esp_gmf_obj_handle_t)tsk), tsk);

[Haerteleric]

I tracked down the cause of the slow audio playback. The issue comes from this line in the mixer implementation:

esp-gmf/elements/gmf_audio/esp_gmf_mixer.c

Line 185 in 94f7c70

int wait_ticks = ((in_port->wait_ticks < (mixer->frame_time / index)) ? (mixer->frame_time / index) : (in_port->wait_ticks));

The original code causes excessive waiting when only a single pipeline is active, which results in ~0.33× playback speed.

I replaced it with the following logic:

int wait_ticks = ((in_port->wait_ticks < (mixer->frame_time / index)) ? 
                  (mixer->frame_time / index) : 
                  (in_port->wait_ticks));

if (in_port->wait_ticks == 0) {
    wait_ticks = 0;
}

With this change, single-pipeline playback now runs at normal speed, even when other pipelines are idle.

[TempoTian]

I see one potential issue with you code, you can change the call order, you need stop pipeline before deinit, maybe it can fix your run timeout issue. If still NG, add some log please, we can provide some analysis for you.

           if(s_tracks[i].pipeline) {
                esp_gmf_pipeline_stop(s_tracks[i].pipeline);
                s_tracks[i].pipeline = NULL;
            }
           if(s_tracks[i].task) {
                esp_gmf_task_deinit(s_tracks[i].task);
                s_tracks[i].task = NULL;
            }
           

[Haerteleric]

yeah i just saw that aswell, but its just the error-exit handler.

[Haerteleric]

do i need to call esp_gmf_pipeline_loading_jobs after esp_gmf_pipeline_reset?

[TempoTian]

Yes, here is the call sequense, you can follow it and have a try:

// Reuse pipeline to run
esp_gmf_task_init(&cfg, &mngr->task[i]);
esp_gmf_pipeline_bind_task(pipeline, mngr->task[i]);
esp_gmf_pipeline_loading_jobs(pipeline);
esp_gmf_pipeline_prev_run(pipeline);// on demand if need run some prepare things ahead
esp_gmf_pipeline_run(pipeline);

// Stop pipeline and release task
esp_gmf_pipeline_prev_stop(pipeline->pipeline); // on demand if need run some clearup ahead, specially for pipelines has dependency
esp_gmf_pipeline_stop(pipeline->pipeline);
esp_gmf_pipeline_reset(pipeline->pipeline);
esp_gmf_pipeline_bind_task(pipeline->pipeline, NULL);
esp_gmf_task_deinit(mngr->task[i]);

[Haerteleric]

Do i need a new task everytime i restarte the pipeline ? Looking at esp_gmf_pipeline_reset The assigned task is reset aswell

[Haerteleric]

So using your code, or just calling
esp_gmf_pipeline_reset andesp_gmf_pipeline_loading_jobs has the same effect.

The pipeline restarts, but has WAY too fast playback speed. my 14 sec clip finishes in less then 1 sec.

[TempoTian]

If you just re-run the task, you can call esp_gmf_pipeline_stop, esp_gmf_pipeline_start repeatly and keep task not destroyed it is OK.

[TempoTian]

As you use file io, when finished you need seek to start to avoid reading data after EOF

esp_gmf_err_t esp_gmf_io_set_pos(esp_gmf_io_handle_t handle, uint64_t byte_pos);

[Haerteleric]

This is allrdy done in

static esp_gmf_err_t _file_close(esp_gmf_io_handle_t io)
{
    file_io_stream_t *file_io = (file_io_stream_t *)io;
    esp_gmf_info_file_t info = {0};
    esp_gmf_io_get_info((esp_gmf_io_handle_t)file_io, &info);
    ESP_LOGI(TAG, "CLose, %p, pos = %d/%d", file_io, (int)info.pos, (int)info.size);
    if (file_io->is_open) {
        close(file_io->file);
        file_io->is_open = false;
    }
    esp_gmf_io_set_pos((esp_gmf_io_handle_t)io, 0);
    return ESP_GMF_ERR_OK;
}

I feel like, the sample_rate of the pipeline is set to 0 after a File is played completely.
And if i restart the pipeline the sample rate is still wrong.

Take a look at this Log (timestamps !!):

I (3450) MICUBO_AUDIO: Playing track 1: /sdcard/A.mp3
Track 1 state: ESP_GMF_EVENT_STATE_INITIALIZED
I (3450) ESP_GMF_FILE: Open, dir:1, uri:/sdcard/A.mp3
I (3450) ESP_GMF_FILE: File size: 140804 byte, file position: 0
I (3450) ESP_GMF_TASK: One times job is complete, del[wk:0x4208cff4, ctx:0x4208b9fc, label:mp3Decoder_open]
I (3460) ESP_GMF_PORT: ACQ IN, new self payload:0x4208cff4, port:0x4208bf90, el:0x4208b9fc-mp3Decoder
W (3480) ESP_GMF_ASMP_DEC: Not enough memory for out, need:2304, old: 1024, new: 2304
I (3480) ESP_GMF_TASK: One times job is complete, del[wk:0x4208e9e8, ctx:0x4208baf8, label:channelConverter_in_open]
I (3490) ESP_GMF_TASK: One times job is complete, del[wk:0x4208ea50, ctx:0x4208bc50, label:bitDepthConverter_open]
I (3500) ESP_GMF_TASK: One times job is complete, del[wk:0x4208e9e8, ctx:0x4208bda4, label:sampleRateConverter_open]
I (16200) ESP_GMF_FILE: No more data, ret: 0
I (16200) ESP_GMF_FILE: No more data, ret: 0
I (16200) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208d02c, job:0x4208b9fc-mp3Decoder_proc]
I (16210) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208ea0c, job:0x4208baf8-channelConverter_in_proc]
I (16220) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208ea74, job:0x4208bc50-bitDepthConverter_proc]
I (16230) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208eab4, job:0x4208bda4-sampleRateConverter_proc]
I (16240) ESP_GMF_FILE: CLose, 0x4208befc, pos = 140804/140804
I (16250) ESP_GMF_TASK: One times job is complete, del[wk:0x4208d02c, ctx:0x4208b9fc, label:mp3Decoder_close]
I (16250) ESP_GMF_TASK: One times job is complete, del[wk:0x4208d050, ctx:0x4208baf8, label:channelConverter_in_close]
I (16260) ESP_GMF_TASK: One times job is complete, del[wk:0x4208e9e8, ctx:0x4208bc50, label:bitDepthConverter_close]
I (16280) ESP_GMF_TASK: One times job is complete, del[wk:0x4208ea0c, ctx:0x4208bda4, label:sampleRateConverter_close]
I (16290) ESP_GMF_TASK: Waiting to run... [tsk:track_1-0x4085f2a0, wk:0x0, run:0]
I (16290) ESP_GMF_TASK: Waiting to run... [tsk:track_1-0x4085f2a0, wk:0x0, run:0]
I (18530) MICUBO_AUDIO: Playing track 1: /sdcard/A.mp3
Track 1 state: ESP_GMF_EVENT_STATE_FINISHED
I (18530) ESP_GMF_TASK: Waiting to run... [tsk:track_1-0x4085f2a0, wk:0x4208d018, run:0]
I (18530) ESP_GMF_FILE: Open, dir:1, uri:/sdcard/A.mp3
I (18540) ESP_GMF_FILE: File size: 140804 byte, file position: 0
I (18540) ESP_GMF_TASK: One times job is complete, del[wk:0x4208d018, ctx:0x4208b9fc, label:mp3Decoder_open]
I (18560) ESP_GMF_TASK: One times job is complete, del[wk:0x4208d018, ctx:0x4208baf8, label:channelConverter_in_open]
I (18560) ESP_GMF_TASK: One times job is complete, del[wk:0x4208ea2c, ctx:0x4208bc50, label:bitDepthConverter_open]
I (18570) ESP_GMF_TASK: One times job is complete, del[wk:0x4208d018, ctx:0x4208bda4, label:sampleRateConverter_open]
I (20170) ESP_GMF_FILE: No more data, ret: 0
I (20180) ESP_GMF_FILE: No more data, ret: 0
I (20180) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208d03c, job:0x4208b9fc-mp3Decoder_proc]
I (20180) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208e9e8, job:0x4208baf8-channelConverter_in_proc]
I (20190) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208ea50, job:0x4208bc50-bitDepthConverter_proc]
I (20200) ESP_GMF_TASK: Job is done, [tsk:track_1-0x4085f2a0, wk:0x4208ea90, job:0x4208bda4-sampleRateConverter_proc]
I (20210) ESP_GMF_FILE: CLose, 0x4208befc, pos = 140804/140804
I (20220) ESP_GMF_TASK: One times job is complete, del[wk:0x4208d018, ctx:0x4208b9fc, label:mp3Decoder_close]
I (20230) ESP_GMF_TASK: One times job is complete, del[wk:0x4208d03c, ctx:0x4208baf8, label:channelConverter_in_close]
I (20240) ESP_GMF_TASK: One times job is complete, del[wk:0x4208e9e8, ctx:0x4208bc50, label:bitDepthConverter_close]
I (20250) ESP_GMF_TASK: One times job is complete, del[wk:0x4208ea0c, ctx:0x4208bda4, label:sampleRateConverter_close]
I (20260) ESP_GMF_TASK: Waiting to run... [tsk:track_1-0x4085f2a0, wk:0x0, run:0]

The First play is normal, the second is WAY to fast

[Haerteleric]

Okay Restart is now working via:

        esp_gmf_pipeline_stop(s_tracks[trackIndex].pipeline);
        esp_gmf_pipeline_reset(s_tracks[trackIndex].pipeline);
        esp_gmf_pipeline_loading_jobs(s_tracks[trackIndex].pipeline)

But ONLY if i restart the pipeline BEFORE The Track finishes playing.

If i reuse the same Pipeline AFTER The Track finishes playing, the second run is way to fast

[Haerteleric]

So it seems like Task the reach ESP_GMF_EVENT_STATE_FINISHED, have no or wrong exec pacing if restarted.