Can I move memory for `audio_client_init` from SRAM to PSRAM?

[Vadimatorik]

Answers checklist.

• I have read the documentation SiFli-SDK Programming Guide and the issue is not addressed there.
• I have updated my SiFli-SDK branch (main or release) to the latest version and checked that the issue is present there.
• I have searched the issue tracker for a similar issue and not found a similar issue.

General issue report

I'm using a 523 microcontroller MP3 player and a large LCD. The LCD needs a line buffer (double) of 100 lines. I don't have SRAM to play music:

08-27 07:40:19:912    Assertion failed at function:audio_client_init, line number:3507 ,(handle->ring_pool)
08-27 07:40:19:913    Previous ISR enable 0
08-27 07:40:19:914    =====================
08-27 07:40:19:915    PSP: 0x20057f54, MSP: 0x200340e0
08-27 07:40:19:915    =====================
08-27 07:40:19:916     sp: 0x20057fb8
08-27 07:40:19:917    psr: 0x40000000
08-27 07:40:19:918    r00: 0x30000000
08-27 07:40:19:919    r01: 0x0000031f
08-27 07:40:19:919    r02: 0x00000017
08-27 07:40:19:920    r03: 0x120e48d5
08-27 07:40:19:921    r12: 0x20000649
08-27 07:40:19:922     lr: 0x1212c6c5
08-27 07:40:19:923     pc: 0x20000654
08-27 07:40:19:924    ===============
08-27 07:40:19:926    ======
08-27 07:40:19:927    fatal error on thread: mp3

I want to use PSRAM to play music. How do I do that?

[yxGC]

You can implement audio_mem_xxx functions(at audio_mem.c) in your own project to move them to PSRAM.

[Vadimatorik]

Great. It worked)

#include <rtthread.h>
#include "mem_section.h"

#define AUDIO_PSRAM_HEAP_SIZE (1024 * 256)

L2_NON_RET_BSS_SECT_BEGIN(AUDIO_PSRAM_HEAP)
ALIGN(4) static uint8_t audio_psram_heap[AUDIO_PSRAM_HEAP_SIZE];
L2_NON_RET_BSS_SECT_END

struct rt_memheap audio_psram_memheap;

#define LOG_PREFIX "[AUDIO_MEM] "
#define LOG(...)   rt_kprintf(LOG_PREFIX __VA_ARGS__)

void audio_memory_init() {
    LOG("Init\n");
    rt_err_t rv = rt_memheap_init(&audio_psram_memheap, "audio_psram_memheap", (void *)audio_psram_heap, AUDIO_PSRAM_HEAP_SIZE);
    RT_ASSERT(rv == RT_EOK);
}

extern "C" {
	
void *audio_mem_malloc(uint32_t size)
{
    void *ptr = rt_memheap_alloc(&audio_psram_memheap, size);
    RT_ASSERT(ptr);
    return ptr;
}

void audio_mem_free(void *ptr)
{
    rt_memheap_free(ptr);
}

void *audio_mem_calloc(uint32_t count, uint32_t size)
{
    void *ptr = rt_memheap_calloc(&audio_psram_memheap, count, size);
    RT_ASSERT(ptr);
    return ptr;
}

void *audio_mem_realloc_do(void *mem_address, unsigned int newsize)
{
    void *ptr = NULL;
    ptr = rt_memheap_realloc(&audio_psram_memheap, mem_address, newsize);
    RT_ASSERT(ptr);
    return ptr;
}

}

However, I had to make some additional changes:

modified:   middleware/audio/audio_local_music/audio_mp3ctrl.c
modified:   middleware/audio/audio_manager/audio_server.c

However, I encountered another problem. If I interrupt audio playback and start it again, the device freezes (without output to the log and crashing)

Here is my class for playing sound:

#include "audio_player.h"

#ifndef BSP_USING_PC_SIMULATOR

extern "C" {
#include "audio_server.h"
}

#define PREFIX   "[AUDIO PLAYER] "
#define LOG(...) rt_kprintf(PREFIX __VA_ARGS__)

bool Audio_player::play(const char* path, uint8_t volume) {
#ifdef DEBUG
    if (volume == 0) {
        return false;
    }
#endif

    LOG("Play: %s\n", path);
    msg_t info{};

    info.cmd            = CMD::PLAY;
    info.param.filename = path;
    info.param.len      = -1;
    info.volume         = (uint16_t)volume * 15 / 100;

    RT_ASSERT(rt_mq_send(mq_, &info, sizeof(msg_t)) == RT_EOK);

    return true;
}

void Audio_player::stop() {
    msg_t info{};
    info.cmd = CMD::STOP;
    RT_ASSERT(rt_mq_send(mq_, &info, sizeof(msg_t)) == RT_EOK);
}

int Audio_player::callback(audio_server_callback_cmt_t cmd, void* obj,
                           uint32_t reserved) {
    if (cmd != as_callback_cmd_play_to_end) {
        return 0;
    }

    auto o = reinterpret_cast<Audio_player*>(obj);

    o->stop();

    return 0;
}


void Audio_player::thread(void* obj) {
    rt_err_t err = RT_ERROR;
    msg_t    msg;
    auto     o = reinterpret_cast<Audio_player*>(obj);

    while (true) {
        rt_mq_recv(o->mq_, &msg, sizeof(msg), RT_WAITING_FOREVER);
        LOG("Receive cmd: %s, volume: %u\n",
            (msg.cmd == CMD::PLAY) ? "play" : "stop", msg.volume);

        if (msg.cmd == CMD::PLAY) {
            if (o->handle_) {
                LOG("Close track\n");
                mp3ctrl_close(o->handle_);
                o->handle_ = nullptr;
            }
            LOG("Open track\n");
            o->handle_ =
                mp3ctrl_open(AUDIO_TYPE_LOCAL_MUSIC, msg.param.filename,
                             Audio_player::callback, obj);
            RT_ASSERT(o->handle_);

            LOG("Set volume\n");
            RT_ASSERT(audio_server_set_private_volume(AUDIO_TYPE_LOCAL_MUSIC,
                                                      msg.volume) == RT_EOK);

            LOG("Play\n");
            mp3ctrl_play(o->handle_);
        } else {
            LOG("Close track\n");
            mp3ctrl_close(o->handle_);
            o->handle_ = nullptr;
        }
    }
}

Audio_player::Audio_player() {
    mq_ = rt_mq_create("mp3_proc_mq", sizeof(msg_t), 60, RT_IPC_FLAG_FIFO);
    RT_ASSERT(mq_);
    thread_ =
        rt_thread_create("mp3_proc", Audio_player::thread, this, 1024 * 2,
                         RT_THREAD_PRIORITY_MIDDLE, RT_THREAD_TICK_DEFAULT);
    RT_ASSERT(thread_);
    RT_ASSERT(rt_thread_startup(thread_) == RT_EOK);
}

#endif

.h file:

#pragma once

#include "rtconfig.h"

#ifndef BSP_USING_PC_SIMULATOR

#include <rtthread.h>

extern "C" {
#include "audio_mp3ctrl.h"
}

class Audio_player {
public:
    Audio_player();

    bool play(const char* path, uint8_t volume);

private:
    void        stop();
    static int  callback(audio_server_callback_cmt_t, void*, uint32_t);
    static void thread(void*);

private:
    enum class CMD {
        PLAY = 0,
        STOP,
    };

    struct msg_t {
        CMD                   cmd;
        mp3_ioctl_cmd_param_t param;
        uint8_t               volume;
    };

private:
    mp3ctrl_handle handle_{};
    rt_thread_t    thread_{};
    rt_mq_t        mq_{};
};

#endif

Here's the normal output (play, wait, play):

08-28 09:54:54:888    [AUDIO PLAYER] Play: volume_change.mp3
08-28 09:54:54:891    [AUDIO PLAYER] Receive cmd: play, volume: 8
08-28 09:54:54:894    [AUDIO PLAYER] Open track
08-28 09:54:54:904    [AUDIO PLAYER] Set volume
08-28 09:54:54:906    [AUDIO PLAYER] Play
08-28 09:54:54:921    [I/drv.audprc] set g_eq_enable=1
08-28 09:54:54:923    [I/drv.audcodec] dac set volume: prc=0 codec=-40
08-28 09:54:54:924    
08-28 09:54:54:925    [I/drv.audcodec] codec dac mute set:1
08-28 09:54:54:927    
08-28 09:54:54:928    [I/drv.audprc] samplerate = 44100, select music eq
08-28 09:54:54:929    
08-28 09:54:54:931    [I/drv.audprc] -----tx dma size = 1600
08-28 09:54:54:932    enable pll 
08-28 09:54:54:933    new PLL_ENABLE vco:8, freq_type:3
08-28 09:54:54:935    pll lock! freq_type:3
08-28 09:54:54:936    [I/drv.audcodec] pll config state:2, samplerate:44100 
08-28 09:54:54:937    
08-28 09:54:54:939    sifli_aw8155 to do power handle 
08-28 09:54:54:940    sifli_aw8155_start,mode:1 10
08-28 09:54:55:129    [I/drv.audcodec] codec dac mute set:0
08-28 09:54:55:131    
08-28 09:54:55:133    [I/drv.audcodec] dac set volume: prc=0 codec=-40
08-28 09:54:55:134    
08-28 09:54:57:048    [AUDIO PLAYER] Receive cmd: stop, volume: 0
08-28 09:54:57:051    [AUDIO PLAYER] Close track
08-28 09:54:57:064    sifli_aw8155_stop 
08-28 09:54:57:080    [I/drv.audcodec] codec dac mute set:1
08-28 09:54:57:082    
08-28 09:54:57:083    [I/drv.audcodec] bf0_audio_stop 0x100
08-28 09:54:57:084    
08-28 09:54:57:085    [I/drv.audcodec] audcodec close dac
08-28 09:54:57:086    [I/drv.audcodec] bf0_audio_stop 0x100 done
08-28 09:54:57:089    
08-28 09:54:57:090    [I/drv.audprc] bf0_audio_stop 0x100
08-28 09:54:57:091    
08-28 09:54:57:092    [I/drv.audprc] audprc close adc/dac
08-28 09:54:57:094    [I/drv.audprc] bf0_audio_stop 0x100 done
08-28 09:54:57:095    
08-28 09:54:57:096    PLL disable

Here is the abnormal output (play, interrupt, play):

08-28 09:56:08:064    [AUDIO PLAYER] Open track
08-28 09:56:08:067    [AUDIO PLAYER] Set volume
08-28 09:56:08:071    [AUDIO PLAYER] Play
08-28 09:56:08:075    [I/drv.audprc] set g_eq_enable=1
08-28 09:56:08:077    [I/drv.audcodec] dac set volume: prc=0 codec=-28
08-28 09:56:08:078    
08-28 09:56:08:079    [I/drv.audcodec] codec dac mute set:1
08-28 09:56:08:079    
08-28 09:56:08:080    [I/drv.audprc] samplerate = 44100, select music eq
08-28 09:56:08:081    
08-28 09:56:08:082    [I/drv.audprc] -----tx dma size = 1600
08-28 09:56:08:083    enable pll 
08-28 09:56:08:084    new PLL_ENABLE vco:8, freq_type:3
08-28 09:56:08:085    pll lock! freq_type:3
08-28 09:56:08:085    [I/drv.audcodec] pll config state:2, samplerate:44100 
08-28 09:56:08:086    
08-28 09:56:08:087    sifli_aw8155 to do power handle 
08-28 09:56:08:088    sifli_aw8155_start,mode:1 10
08-28 09:56:08:274    [I/drv.audcodec] codec dac mute set:0
08-28 09:56:08:276    
08-28 09:56:08:277    [I/drv.audcodec] dac set volume: prc=0 codec=-28
08-28 09:56:08:278    
08-28 09:56:08:599    [AUDIO PLAYER] Play: volume_change.mp3
08-28 09:56:08:605    [AUDIO PLAYER] Receive cmd: play, volume: 3
08-28 09:56:08:608    [AUDIO PLAYER] Close track
08-28 09:56:08:632    [I/drv.audcodec] dac set volume: prc=0 codec=-48
08-28 09:56:08:633    
08-28 09:56:08:648    [I/drv.audcodec] dac set volume: prc=0 codec=-64
08-28 09:56:08:649    
08-28 09:56:08:760    sifli_aw8155_stop 
08-28 09:56:08:761    [I/drv.audcodec] codec dac mute set:1
08-28 09:56:08:762    
08-28 09:56:08:763    [I/drv.audcodec] bf0_audio_stop 0x100
08-28 09:56:08:763    
08-28 09:56:08:765    [I/drv.audcodec] audcodec close dac
08-28 09:56:08:765    [I/drv.audcodec] bf0_audio_stop 0x100 done
08-28 09:56:08:766    
08-28 09:56:08:767    [I/drv.audprc] bf0_audio_stop 0x100
08-28 09:56:08:768    
08-28 09:56:08:769    [I/drv.audprc] audprc close adc/dac
08-28 09:56:08:770    [I/drv.audprc] bf0_audio_stop 0x100 done
08-28 09:56:08:771    
08-28 09:56:08:772    PLL disable
08-28 09:56:08:772    [AUDIO PLAYER] Open track
08-28 09:56:08:777    [AUDIO PLAYER] Set volume
08-28 09:56:08:780    [AUDIO PLAYER] Play

[Vadimatorik]

Do you have any idea what might be causing it to hang? Sometimes playback freezes even after a pause.

[yxGC]

Does it introduced by moving memory from SRAM to PSRAM?

[Vadimatorik]

I think so. I didn't have this problem before. Maybe it's because of using a different memory manager. I'll try using the current memory manager and SRAM.

[Vadimatorik]

In SRAM a I have it:

08-28 10:31:28:118    Assertion failed at function:mp3ctrl_thread_entry_file, line number:565 ,(hMP3Decoder)

This uses malloc. I think it's not right

[Vadimatorik]

I reduced my own buffer in SRAM and tried again. The problem was repeated even in SRAM

[Vadimatorik]

I tried to revoke my SDK changes (use a bunch from rt thread). There was a problem from the very beginning. I haven't checked this after updating the SDK with the old version.

[Vadimatorik]

Can you try it for yourself? It requires stopping (completely) and starting (from the beginning) the audio many times at random times. This will allow you to repeat the problem.

[ljq-ctrl]

it'll try it,
int these two files（audio_mp3ctrl.c and audio_server.c)

can you define these after include all head files

#undef LOG_I
#define LOG_I(...) rt_kprintf(VA_ARGS)

I want to see the log in these files

[ljq-ctrl]

i chang audio_mem_* as you,

--- a/example/multimedia/audio/local_music/project/proj.conf
+++ b/example/multimedia/audio/local_music/project/proj.conf
@@ -1,4 +1,8 @@
+CONFIG_RT_USING_DFS_ELMFAT=y
+CONFIG_RT_USING_ULOG=y
+CONFIG_ULOG_OUTPUT_LVL_I=y
+CONFIG_ULOG_USING_ISR_LOG=y
CONFIG_BSP_USING_FULL_ASSERT=y
CONFIG_AUDIO=y
-CONFIG_RT_USING_DFS_ELMFAT=y
CONFIG_AUDIO_LOCAL_MUSIC=y
+CONFIG_RT_USING_MEMHEAP=y

--- a/external/libhelix-mp3/real/buffers.c
+++ b/external/libhelix-mp3/real/buffers.c
@@ -44,11 +44,18 @@

• to use other the default system malloc() and free() for heap management this is
• the only file you'll need to change.
  **************************************************************************************/

+#include "rtthread.h"
//#include "hlxclib/stdlib.h" /* for malloc, free */
#include <stdlib.h>
#include <string.h>
#include "coder.h"
+#include "audio_mem.h"
+#undef malloc
+#undef free
+#define malloc audio_mem_malloc
+#define free audio_mem_free
+

--- a/middleware/audio/audio_manager/audio_server.c
+++ b/middleware/audio/audio_manager/audio_server.c
@@ -95,12 +95,15 @@

#define START_RX_IN_TX_INTERUPT 1
#define PDM_DEVICE_NAME "pdm1"
+
+#if 0
#undef audio_mem_malloc
#undef audio_mem_free
#undef audio_mem_calloc
#define audio_mem_malloc rt_malloc
#define audio_mem_free rt_free
#define audio_mem_calloc rt_calloc
+#endif

--- a/middleware/audio/audio_local_music/audio_mp3ctrl.c
+++ b/middleware/audio/audio_local_music/audio_mp3ctrl.c
@@ -195,6 +195,7 @@ typedef struct ID3v2
char size[4]; // size, not include id3v2_t
} id3v2_t;

+#if 0
#undef audio_mem_malloc
#undef audio_mem_free
#undef audio_mem_calloc

#ifdef SOLUTION_WATCH
#define audio_mem_malloc app_malloc
#define audio_mem_free app_free
#define audio_mem_calloc app_calloc
#else
#define audio_mem_malloc rt_malloc
#define audio_mem_free rt_free
#define audio_mem_calloc rt_calloc
#endif
+#endif

// ---------------------
// test program using you Audio_player class, it work well for more thbn 2 hours

void main()
{
LOG("test\n");
audio_memory_init();
Audio_player player;
srand(rt_tick_get());
while (1)
{
for (int i = 0; i < 50; i++)
{
player.play("/Meet_16k.mp3", 15);
}
for (int i = 0; i < 100; i++)
{
player.play("/Meet_16k.mp3", 15);
rt_thread_mdelay(i);
}
for (int i = 0; i < 100; i++)
{
player.play("/Meet_16k.mp3", 15);
rt_thread_mdelay(rand() % 1000);
}
rt_thread_mdelay(5000);
}
}

---

so can you enable LOG_I as I changed in proj.conf, these can see more log in audio_server.c and audio_mpctrl.c，
or give a system memory dump and symbols file under project build dir.

[ljq-ctrl]

It has been working normally from last night to this morning, more than 12 hours。 maybe need a dump to analysis

[ljq-ctrl]

when using PSRAM, should add CONFIG_PSRAM_CACHE_WB=y in project config file for high cpu speed, otherwise low speed than using sram.