Added media_stream component which would be used to record/play audio_video

Author: vikramdattu

esp_port/components/media_stream/CMakeLists.txt

@@ -0,0 +1,25 @@
+if (ENABLE_SIGNALLING_ONLY)
+    set(srcs )
+else()
+    set(srcs
+        "src/esp_video_if.c"
+        "src/esp_h264_hw_enc.c"
+        "src/app_camera_esp.c"
+        "src/esp32p4_frame_grabber.c"
+        "src/H264FrameGrabber.c"
+        "src/OpusFrameGrabber.c"
+        "src/OpusAudioPlayer.c"
+        "src/resampling.c"
+        "src/ringbuf.c"
+        "src/video_capture_adapter.c"
+        "src/audio_capture_adapter.c"
+        "src/audio_player_adapter.c"
+        "src/video_player_adapter.c"
+    )
+endif()
+
+idf_component_register(
+    SRCS ${srcs}
+    INCLUDE_DIRS "include"
+    PRIV_INCLUDE_DIRS "src"
+)

esp_port/components/media_stream/Kconfig.projbuild

@@ -0,0 +1,14 @@
+menu "ESP Video Interface"
+    config USE_ESP_VIDEO_IF
+        bool "Use ESP Video Interface"
+        default y
+    help
+        Use ESP Video Interface for video capture.
+
+    config ESP_VIDEO_IF_HOR_FLIP
+        bool "Horizontal Flip"
+        default y
+        depends on USE_ESP_VIDEO_IF
+    help
+        Flip the image horizontally.
+endmenu

esp_port/components/media_stream/README.md

@@ -0,0 +1,13 @@
+# Media Stream
+
+This component provides a media stream interface for the KVS SDK for ESP.
+
+## Components
+
+- `OpusAudioPlayer`: Plays a single frame of OPUS. Internally, the frame is decoded and inserted into a ring buffer. The special i2s task is used to read the ring buffer and play the audio.
+- `H264FrameGrabber`: Grabs one h264 encoded frame. Camera frames are captured, encoded using H264 encoder and continuously inserted into a frame_queue.
+- `OpusFrameGrabber`: Grabs one frame of OPUS from the frame queue. The i2s data is recorded by the module in a ring buffer, encoded using the Opus encoder and continuously inserted into a frame_queue.
+
+## Usage
+
+- Please refer to the header files in media_stream/include for the API descriptions.

esp_port/components/media_stream/idf_component.yml

@@ -0,0 +1,36 @@
+dependencies:
+  espressif/esp_video:
+    version: "*"
+    rules:
+      - if: "target in [esp32p4]"
+
+  espressif/esp_h264:
+    public: true
+    matches:
+      - if: "target in [esp32s3]"
+        version: "~0.1.1"
+      - if: "target in [esp32p4]"
+        version: "1.0.4"
+
+  espressif/esp_audio_codec:
+    version: "*"
+
+  espressif/esp_codec_dev:
+    version: "*"
+
+  espressif/esp32-camera:
+    version: ^2.0.2
+
+  # local components
+  esp32_p4_function_ev_board:
+    path: ../esp32_p4_function_ev_board
+    version: "*"
+    rules:
+      - if: "target in [esp32p4]"
+
+  esp_webrtc_utils:
+    path: ../esp_webrtc_utils
+    version: "*"
+
+description: Media stream component
+version: "0.0.8"

esp_port/components/media_stream/include/H264FrameGrabber.h

@@ -0,0 +1,32 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @brief grab camera frames and encode them with h264 encoder
+ *
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include <esp_h264_types.h>
+
+#define ENCODER_TASK        (1)
+
+#if ENCODER_TASK
+typedef struct {
+    uint8_t *buffer; /*<! Data buffer */
+    uint32_t len;    /*<! It is buffer length in byte */
+    esp_h264_frame_type_t type; /* Frame type */
+} esp_h264_out_buf_t;
+
+esp_h264_out_buf_t *get_h264_encoded_frame();
+
+/* Explicitly initialize camera and encoder */
+void camera_and_encoder_init(void);
+#else
+void get_h264_encoded_frame(uint8_t *out_buf, uint32_t *frame_len);
+#endif

esp_port/components/media_stream/include/OpusAudioPlayer.h

@@ -0,0 +1,33 @@
+/**
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "esp_err.h"
+#include "inttypes.h"
+
+#pragma once
+
+/**
+ * @brief Initialize OPUS audio player
+ *
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t OpusAudioPlayerInit();
+
+/**
+ * @brief Deinitialize OPUS audio player
+ *
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t OpusAudioPlayerDeinit();
+
+/**
+ * @brief Decode one frame of OPUS to PCM
+ *
+ * @param data Pointer to encoded OPUS data
+ * @param size Size of encoded OPUS data
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t OpusAudioPlayerDecode(uint8_t *data, size_t size);

esp_port/components/media_stream/include/OpusFrameGrabber.h

@@ -0,0 +1,21 @@
+/*
+ * SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @brief grab mic data encode them with opus encoder
+ *
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+typedef struct {
+    uint8_t *buffer; /*<! Data buffer */
+    uint32_t len;    /*<! It is buffer length in byte */
+} esp_opus_out_buf_t;
+
+esp_opus_out_buf_t *get_opus_encoded_frame();

esp_port/components/media_stream/include/audio_capture.h

@@ -0,0 +1,118 @@
+/*
+ * SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @brief Generic audio capture interface for microphone input and encoding
+ */
+
+#pragma once
+
+#include <stdint.h>
+#include "esp_err.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Audio codec type enumeration
+ */
+typedef enum {
+    AUDIO_CODEC_OPUS,
+    AUDIO_CODEC_PCM,
+    AUDIO_CODEC_AAC,
+    /* Add more codecs as needed */
+} audio_codec_type_t;
+
+/**
+ * @brief Audio format configuration
+ */
+typedef struct {
+    uint32_t sample_rate;    /* Sample rate in Hz (e.g., 16000, 44100) */
+    uint8_t channels;        /* Number of channels (1=mono, 2=stereo) */
+    uint16_t bits_per_sample; /* Bits per sample (8, 16, 24, 32) */
+} audio_format_t;
+
+/**
+ * @brief Audio capture configuration
+ */
+typedef struct {
+    audio_codec_type_t codec;     /* Audio codec to use */
+    audio_format_t format;        /* Audio format */
+    uint32_t bitrate;             /* Target bitrate in kbps (for compressed formats) */
+    uint16_t frame_duration_ms;   /* Frame duration in milliseconds */
+    void *codec_specific;         /* Codec-specific parameters if needed */
+} audio_capture_config_t;
+
+/**
+ * @brief Audio frame buffer structure
+ */
+typedef struct {
+    uint8_t *buffer;        /* Data buffer */
+    uint32_t len;           /* Buffer length in bytes */
+    uint64_t timestamp;     /* Timestamp in microseconds */
+} audio_frame_t;
+
+/**
+ * @brief Audio capture handle
+ */
+typedef void* audio_capture_handle_t;
+
+/**
+ * @brief Initialize audio capture with specified configuration
+ *
+ * @param config Audio capture configuration
+ * @param ret_handle Pointer to store the created handle
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t audio_capture_init(audio_capture_config_t *config, audio_capture_handle_t *ret_handle);
+
+/**
+ * @brief Start audio capture
+ *
+ * @param handle Audio capture handle
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t audio_capture_start(audio_capture_handle_t handle);
+
+/**
+ * @brief Stop audio capture
+ *
+ * @param handle Audio capture handle
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t audio_capture_stop(audio_capture_handle_t handle);
+
+/**
+ * @brief Get the next captured audio frame
+ *
+ * @param handle Audio capture handle
+ * @param frame Pointer to store the audio frame
+ * @param wait_ms Time to wait for a frame in milliseconds (0 for non-blocking)
+ * @return esp_err_t ESP_OK on success, ESP_ERR_TIMEOUT if no frame available, otherwise an error code
+ */
+esp_err_t audio_capture_get_frame(audio_capture_handle_t handle, audio_frame_t **frame, uint32_t wait_ms);
+
+/**
+ * @brief Release an audio frame when no longer needed
+ *
+ * @param handle Audio capture handle
+ * @param frame Frame to release
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t audio_capture_release_frame(audio_capture_handle_t handle, audio_frame_t *frame);
+
+/**
+ * @brief Deinitialize audio capture and free resources
+ *
+ * @param handle Audio capture handle
+ * @return esp_err_t ESP_OK on success, otherwise an error code
+ */
+esp_err_t audio_capture_deinit(audio_capture_handle_t handle);
+
+#ifdef __cplusplus
+}
+#endif