Skip to content

Adds Opus codec config support #431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
+64 −11
Open

Adds Opus codec config support #431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a2ee2ad
add opus codec config support
danrossi Jul 12, 2025
d905852
correct codec config type
danrossi Jul 12, 2025
bfdeb4d
Merge branch 'WebAV-Tech:main' into opus-codec
danrossi Jul 13, 2025
7657582
fix typo with codec type map
danrossi Jul 13, 2025
2051f97
Merge branch 'WebAV-Tech:main' into opus-codec
danrossi Jul 15, 2025
cabe10e
Add hardware preference to support check due to AV! hardware encoding…
danrossi Jul 15, 2025
60b044f
create a codec info map for types, codec string and code
danrossi Jul 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 23 additions & 6 deletions packages/av-cliper/src/combinator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Log, EventTool, file2stream, recodemux } from '@webav/internal-utils';
import { OffscreenSprite } from './sprite/offscreen-sprite';
import { EventTool, file2stream, Log, recodemux } from '@webav/internal-utils';
import { sleep } from './av-utils';
import { DEFAULT_AUDIO_CONF } from './clips';
import { OffscreenSprite } from './sprite/offscreen-sprite';

export interface ICombinatorOpts {
width?: number;
Expand All @@ -10,6 +10,8 @@ export interface ICombinatorOpts {
fps?: number;
bgColor?: string;
videoCodec?: string;
audioCodec?: 'opus' | 'aac';
opusConfig?: object;
Copy link
Collaborator

@hughfenghen hughfenghen Jul 13, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is recommended to change the audio field type to

export interface ICombinatorOpts {
  width?: number;
  height?: number;
  bitrate?: number;
  fps?: number;
  bgColor?: string;
  videoCodec?: string;
   /**
   * When specifying a composite video file,
   * setting ⁠`audio: false` will ignore the audio track;
   * if you want to include audio, you can configure the codec and other parameters via ⁠`audio: { codec: 'aac' | 'opus'; opusConfig?: OpusEncoderConfig }`.
   * The default value is `audio: { dodec: ‘aac’ }`
   */
  audio?: false | { codec: 'aac' | 'opus'; opusConfig?: OpusEncoderConfig };
  /**
   * 向输出的视频中写入 meta tags 数据
   */
  metaDataTags?: Record<string, string>;
  /**
   * 不安全，随时可能废弃
   */
  __unsafe_hardwareAcceleration__?: HardwarePreference;
}

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then remove the audioCodec and opusConfig fields

/**
* false 合成的视频文件中排除音轨
*/
Expand Down Expand Up @@ -66,6 +68,8 @@ export class Combinator {
static async isSupported(
args: {
videoCodec?: string;
audioCodec?: string;
hardwareAcceleration?: HardwarePreference;
width?: number;
height?: number;
bitrate?: number;
Expand All @@ -82,6 +86,7 @@ export class Combinator {
((
await self.VideoEncoder.isConfigSupported({
codec: args.videoCodec ?? 'avc1.42E032',
hardwareAcceleration: args.hardwareAcceleration ?? 'no-preference',
width: args.width ?? 1920,
height: args.height ?? 1080,
bitrate: args.bitrate ?? 7e6,
Expand All @@ -90,7 +95,7 @@ export class Combinator {
false) &&
(
await self.AudioEncoder.isConfigSupported({
codec: DEFAULT_AUDIO_CONF.codec,
codec: args.audioCodec ?? DEFAULT_AUDIO_CONF.codec,
sampleRate: DEFAULT_AUDIO_CONF.sampleRate,
numberOfChannels: DEFAULT_AUDIO_CONF.channelCount,
})
Expand Down Expand Up @@ -139,6 +144,8 @@ export class Combinator {
width: 0,
height: 0,
videoCodec: 'avc1.42E032',
audioCodec: 'aac',
opusConfig: {},
audio: true,
bitrate: 5e6,
fps: 30,
Expand Down Expand Up @@ -177,8 +184,17 @@ export class Combinator {
}

#startRecodeMux(duration: number) {
const { fps, width, height, videoCodec, bitrate, audio, metaDataTags } =
this.#opts;
const {
fps,
width,
height,
videoCodec,
audioCodec,
opusConfig,
bitrate,
audio,
metaDataTags,
} = this.#opts;
const recodeMuxer = recodemux({
video: this.#hasVideoTrack
? {
Expand All @@ -195,7 +211,8 @@ export class Combinator {
audio === false
? null
: {
codec: 'aac',
codec: audioCodec,
opusConfig: opusConfig,
sampleRate: DEFAULT_AUDIO_CONF.sampleRate,
channelCount: DEFAULT_AUDIO_CONF.channelCount,
},
Expand Down
46 changes: 41 additions & 5 deletions packages/internal-utils/src/recodemux.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ interface IRecodeMuxOpts {
*/
audio: {
codec: 'opus' | 'aac';
opusConfig: object;
sampleRate: number;
channelCount: number;
} | null;
Expand Down Expand Up @@ -376,17 +377,42 @@ function createVideoEncoder(
return encoder;
}

//codec info map entry
interface AudioCodecInfoEntry {
type: string;
codecString: string;
code: number;
}

//codec info map to type, codec string and codec code
const codecInfoMap: Map<string, AudioCodecInfoEntry> = new Map([
[
'aac',
{
type: 'mp4a',
codecString: 'mp4a.40.2',
code: 0x40,
} as AudioCodecInfoEntry,
],
[
'opus',
{ type: 'Opus', codecString: 'opus', code: 0xad } as AudioCodecInfoEntry,
],
]);

function encodeAudioTrack(
audioOpts: NonNullable<IRecodeMuxOpts['audio']>,
mp4File: MP4File,
avSyncEvtTool: EventTool<Record<'VideoReady' | 'AudioReady', () => void>>,
): AudioEncoder {
const codecInfoMapEntry = codecInfoMap.get(audioOpts.codec)!;
const audioTrackOpts = {
timescale: 1e6,
samplerate: audioOpts.sampleRate,
channel_count: audioOpts.channelCount,
hdlr: 'soun',
type: audioOpts.codec === 'aac' ? 'mp4a' : 'Opus',
//map codec to type
type: codecInfoMapEntry.type,
name: 'Track created with WebAV',
};

Expand All @@ -403,7 +429,9 @@ function encodeAudioTrack(
});

const encoderConf = {
codec: audioOpts.codec === 'aac' ? 'mp4a.40.2' : 'opus',
//map codec to AudioEncoder codec
codec: codecInfoMapEntry.codecString,
opus: audioOpts.opusConfig,
sampleRate: audioOpts.sampleRate,
numberOfChannels: audioOpts.channelCount,
bitrate: 128_000,
Expand All @@ -424,9 +452,13 @@ function encodeAudioTrack(
if (trackId === -1) {
// 某些设备不会输出 description
const desc = meta?.decoderConfig?.description;

trackId = mp4File.addTrack({
...audioTrackOpts,
description: desc == null ? undefined : createESDSBox(desc),
description:
desc == null
? undefined
: createESDSBox(desc, codecInfoMapEntry.code),
});
avSyncEvtTool.emit('AudioReady');
Log.info('AudioEncoder, audio track ready, trackId:', trackId);
Expand All @@ -449,9 +481,13 @@ function encodeAudioTrack(
* 创建 ESDS 盒子（MPEG-4 Elementary Stream Descriptor）
* ESDS 盒子用于描述 MPEG-4 的流信息，如编解码器类型、流类型、最大比特率、平均比特率等
* @param config - 配置信息，可以是 `ArrayBuffer` 或 `ArrayBufferView` 类型
* @param codecConfig The audio codec code for m4a and opus
* @return 返回一个 ESDS box
*/
function createESDSBox(config: ArrayBuffer | ArrayBufferView) {
function createESDSBox(
config: ArrayBuffer | ArrayBufferView,
codecCode: number,
) {
const configlen = config.byteLength;
const buf = new Uint8Array([
0x00, // version 0
Expand All @@ -468,7 +504,7 @@ function createESDSBox(config: ArrayBuffer | ArrayBufferView) {

0x04, // descriptor_type
0x12 + configlen, // length
0x40, // codec : mpeg4_audio
codecCode, // codec : mpeg4_audio
0x15, // stream_type
0x00,
0x00,
Expand Down
Loading