From a2ee2ad1ec68db79eb9165890a50a767a8eb5e89 Mon Sep 17 00:00:00 2001 From: Daniel Rossi Date: Sat, 12 Jul 2025 23:43:15 +1000 Subject: [PATCH 1/5] add opus codec config support add custom opus codec support --- packages/av-cliper/src/combinator.ts | 21 +++++++++++++++++---- packages/internal-utils/src/recodemux.ts | 18 ++++++++++++++++-- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/av-cliper/src/combinator.ts b/packages/av-cliper/src/combinator.ts index 197b6966..156fc3cf 100644 --- a/packages/av-cliper/src/combinator.ts +++ b/packages/av-cliper/src/combinator.ts @@ -10,6 +10,8 @@ export interface ICombinatorOpts { fps?: number; bgColor?: string; videoCodec?: string; + audioCodec?: string; + opusConfig?: object; /** * false 合成的视频文件中排除音轨 */ @@ -66,6 +68,7 @@ export class Combinator { static async isSupported( args: { videoCodec?: string; + audioCodec?: string; width?: number; height?: number; bitrate?: number; @@ -88,13 +91,20 @@ export class Combinator { }) ).supported ?? false) && - ( + (( await self.AudioEncoder.isConfigSupported({ codec: DEFAULT_AUDIO_CONF.codec, sampleRate: DEFAULT_AUDIO_CONF.sampleRate, numberOfChannels: DEFAULT_AUDIO_CONF.channelCount, }) - ).supported) ?? + ).supported) || + ( + await self.AudioEncoder.isConfigSupported({ + codec: args.audioCodec, + sampleRate: DEFAULT_AUDIO_CONF.sampleRate, + numberOfChannels: DEFAULT_AUDIO_CONF.channelCount, + }) + ).supported)) ?? false ); } @@ -139,6 +149,8 @@ export class Combinator { width: 0, height: 0, videoCodec: 'avc1.42E032', + audioCodec: 'aac', + opusConfig: {}, audio: true, bitrate: 5e6, fps: 30, @@ -177,7 +189,7 @@ export class Combinator { } #startRecodeMux(duration: number) { - const { fps, width, height, videoCodec, bitrate, audio, metaDataTags } = + const { fps, width, height, videoCodec, audioCodec, opusConfig, bitrate, audio, metaDataTags } = this.#opts; const recodeMuxer = recodemux({ video: this.#hasVideoTrack @@ -195,7 +207,8 @@ export class Combinator { audio === false ? null : { - codec: 'aac', + codec: audioCodec, + opusConfig: opusConfig, sampleRate: DEFAULT_AUDIO_CONF.sampleRate, channelCount: DEFAULT_AUDIO_CONF.channelCount, }, diff --git a/packages/internal-utils/src/recodemux.ts b/packages/internal-utils/src/recodemux.ts index 9588f717..53282efa 100644 --- a/packages/internal-utils/src/recodemux.ts +++ b/packages/internal-utils/src/recodemux.ts @@ -29,6 +29,7 @@ interface IRecodeMuxOpts { */ audio: { codec: 'opus' | 'aac'; + opusConfig: object; sampleRate: number; channelCount: number; } | null; @@ -358,6 +359,16 @@ function createVideoEncoder( return encoder; } +//codec mapping +const codecTypeMap = { + 'aac': "m4a", + 'opus': "Opus" +}, +codecMap = { + 'aac': "mp4a.40.2", + 'opus': "opus" +}; + function encodeAudioTrack( audioOpts: NonNullable, mp4File: MP4File, @@ -368,7 +379,8 @@ function encodeAudioTrack( samplerate: audioOpts.sampleRate, channel_count: audioOpts.channelCount, hdlr: 'soun', - type: audioOpts.codec === 'aac' ? 'mp4a' : 'Opus', + //map codec to type + type: codecTypeMap[audioOpts.codec], name: 'Track created with WebAV', }; @@ -385,7 +397,9 @@ function encodeAudioTrack( }); const encoderConf = { - codec: audioOpts.codec === 'aac' ? 'mp4a.40.2' : 'opus', + //map codec to AudioEncoder codec + codec: codecMap[audioOpts.codec], + opus: audioOpts.opusConfig, sampleRate: audioOpts.sampleRate, numberOfChannels: audioOpts.channelCount, bitrate: 128_000, From d9058527f8d0f29478c54d00660045f01427f9a8 Mon Sep 17 00:00:00 2001 From: Daniel Rossi Date: Sun, 13 Jul 2025 01:04:26 +1000 Subject: [PATCH 2/5] correct codec config type --- packages/av-cliper/src/combinator.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/av-cliper/src/combinator.ts b/packages/av-cliper/src/combinator.ts index 156fc3cf..039614c1 100644 --- a/packages/av-cliper/src/combinator.ts +++ b/packages/av-cliper/src/combinator.ts @@ -10,7 +10,7 @@ export interface ICombinatorOpts { fps?: number; bgColor?: string; videoCodec?: string; - audioCodec?: string; + audioCodec?: 'opus' | 'aac'; opusConfig?: object; /** * false 合成的视频文件中排除音轨 @@ -91,20 +91,13 @@ export class Combinator { }) ).supported ?? false) && - (( - await self.AudioEncoder.isConfigSupported({ - codec: DEFAULT_AUDIO_CONF.codec, - sampleRate: DEFAULT_AUDIO_CONF.sampleRate, - numberOfChannels: DEFAULT_AUDIO_CONF.channelCount, - }) - ).supported) || ( await self.AudioEncoder.isConfigSupported({ - codec: args.audioCodec, + codec: args.audioCodec ?? DEFAULT_AUDIO_CONF.codec, sampleRate: DEFAULT_AUDIO_CONF.sampleRate, numberOfChannels: DEFAULT_AUDIO_CONF.channelCount, }) - ).supported)) ?? + ).supported) ?? false ); } From 7657582e670f73fa2d35faafdc39bb814e0d724b Mon Sep 17 00:00:00 2001 From: Daniel Rossi Date: Sun, 13 Jul 2025 22:37:24 +1000 Subject: [PATCH 3/5] fix typo with codec type map --- packages/internal-utils/src/recodemux.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/internal-utils/src/recodemux.ts b/packages/internal-utils/src/recodemux.ts index 3644f676..aba54a1f 100644 --- a/packages/internal-utils/src/recodemux.ts +++ b/packages/internal-utils/src/recodemux.ts @@ -361,7 +361,7 @@ function createVideoEncoder( //codec mapping const codecTypeMap = { - 'aac': "m4a", + 'aac': "mp4a", 'opus': "Opus" }, codecMap = { From cabe10ea082f09d73951ef1f6c1d7163a4380469 Mon Sep 17 00:00:00 2001 From: Daniel Rossi Date: Tue, 15 Jul 2025 19:59:37 +1000 Subject: [PATCH 4/5] Add hardware preference to support check due to AV! hardware encoding only supported on certain GPU models. --- packages/av-cliper/src/combinator.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/av-cliper/src/combinator.ts b/packages/av-cliper/src/combinator.ts index 039614c1..7d0240f7 100644 --- a/packages/av-cliper/src/combinator.ts +++ b/packages/av-cliper/src/combinator.ts @@ -1,7 +1,7 @@ -import { Log, EventTool, file2stream, recodemux } from '@webav/internal-utils'; -import { OffscreenSprite } from './sprite/offscreen-sprite'; +import { EventTool, file2stream, Log, recodemux } from '@webav/internal-utils'; import { sleep } from './av-utils'; import { DEFAULT_AUDIO_CONF } from './clips'; +import { OffscreenSprite } from './sprite/offscreen-sprite'; export interface ICombinatorOpts { width?: number; @@ -69,6 +69,7 @@ export class Combinator { args: { videoCodec?: string; audioCodec?: string; + hardwareAcceleration?: HardwarePreference; width?: number; height?: number; bitrate?: number; @@ -85,6 +86,7 @@ export class Combinator { (( await self.VideoEncoder.isConfigSupported({ codec: args.videoCodec ?? 'avc1.42E032', + hardwareAcceleration: args.hardwareAcceleration ?? 'no-preference', width: args.width ?? 1920, height: args.height ?? 1080, bitrate: args.bitrate ?? 7e6, @@ -182,8 +184,17 @@ export class Combinator { } #startRecodeMux(duration: number) { - const { fps, width, height, videoCodec, audioCodec, opusConfig, bitrate, audio, metaDataTags } = - this.#opts; + const { + fps, + width, + height, + videoCodec, + audioCodec, + opusConfig, + bitrate, + audio, + metaDataTags, + } = this.#opts; const recodeMuxer = recodemux({ video: this.#hasVideoTrack ? { From 60b044f0d65186a2ab07277c0e49e0fd92d2f242 Mon Sep 17 00:00:00 2001 From: Daniel Rossi Date: Wed, 16 Jul 2025 02:34:11 +1000 Subject: [PATCH 5/5] create a codec info map for types, codec string and code add codec code map to ESDS --- packages/internal-utils/src/recodemux.ts | 50 +++++++++++++++++------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/packages/internal-utils/src/recodemux.ts b/packages/internal-utils/src/recodemux.ts index 852eccfa..c093ea55 100644 --- a/packages/internal-utils/src/recodemux.ts +++ b/packages/internal-utils/src/recodemux.ts @@ -377,28 +377,42 @@ function createVideoEncoder( return encoder; } -//codec mapping -const codecTypeMap = { - 'aac': "mp4a", - 'opus': "Opus" -}, -codecMap = { - 'aac': "mp4a.40.2", - 'opus': "opus" -}; +//codec info map entry +interface AudioCodecInfoEntry { + type: string; + codecString: string; + code: number; +} + +//codec info map to type, codec string and codec code +const codecInfoMap: Map = new Map([ + [ + 'aac', + { + type: 'mp4a', + codecString: 'mp4a.40.2', + code: 0x40, + } as AudioCodecInfoEntry, + ], + [ + 'opus', + { type: 'Opus', codecString: 'opus', code: 0xad } as AudioCodecInfoEntry, + ], +]); function encodeAudioTrack( audioOpts: NonNullable, mp4File: MP4File, avSyncEvtTool: EventTool void>>, ): AudioEncoder { + const codecInfoMapEntry = codecInfoMap.get(audioOpts.codec)!; const audioTrackOpts = { timescale: 1e6, samplerate: audioOpts.sampleRate, channel_count: audioOpts.channelCount, hdlr: 'soun', //map codec to type - type: codecTypeMap[audioOpts.codec], + type: codecInfoMapEntry.type, name: 'Track created with WebAV', }; @@ -416,7 +430,7 @@ function encodeAudioTrack( const encoderConf = { //map codec to AudioEncoder codec - codec: codecMap[audioOpts.codec], + codec: codecInfoMapEntry.codecString, opus: audioOpts.opusConfig, sampleRate: audioOpts.sampleRate, numberOfChannels: audioOpts.channelCount, @@ -438,9 +452,13 @@ function encodeAudioTrack( if (trackId === -1) { // 某些设备不会输出 description const desc = meta?.decoderConfig?.description; + trackId = mp4File.addTrack({ ...audioTrackOpts, - description: desc == null ? undefined : createESDSBox(desc), + description: + desc == null + ? undefined + : createESDSBox(desc, codecInfoMapEntry.code), }); avSyncEvtTool.emit('AudioReady'); Log.info('AudioEncoder, audio track ready, trackId:', trackId); @@ -463,9 +481,13 @@ function encodeAudioTrack( * 创建 ESDS 盒子（MPEG-4 Elementary Stream Descriptor） * ESDS 盒子用于描述 MPEG-4 的流信息，如编解码器类型、流类型、最大比特率、平均比特率等 * @param config - 配置信息，可以是 `ArrayBuffer` 或 `ArrayBufferView` 类型 + * @param codecConfig The audio codec code for m4a and opus * @return 返回一个 ESDS box */ -function createESDSBox(config: ArrayBuffer | ArrayBufferView) { +function createESDSBox( + config: ArrayBuffer | ArrayBufferView, + codecCode: number, +) { const configlen = config.byteLength; const buf = new Uint8Array([ 0x00, // version 0 @@ -482,7 +504,7 @@ function createESDSBox(config: ArrayBuffer | ArrayBufferView) { 0x04, // descriptor_type 0x12 + configlen, // length - 0x40, // codec : mpeg4_audio + codecCode, // codec : mpeg4_audio 0x15, // stream_type 0x00, 0x00,