Bug 1891082 - Enable decoder reconfiguration in DecoderAgent for AAC/ADTS data through MediaChangeMonitor r=media-playback-reviewers,padenot

marco-c · marco-c · commit e6fa04048aea · 2024-09-16T00:52:52.000Z
This patch enhances `DecoderAgent` by enabling it to reconfigure the underlying decoder when processing *AAC/ADTS* data via `MediaChangeMonitor`. In cases where the `description` field is absent in the `AudioDecoderConfig` for `mp4a.*` (*AAC*) codec, the data stream is assumed to be in *ADTS* format. However, some websites provide raw AAC data under these circumstances, leading to decoding failure since the underlying decoder requires AAC-specific configuration. This patch resolves the issue by allowing the underling decoder to be reconfigured - recreated and reinitialized - through `MediaChangeMonitor` whenever an AAV format change is detected. This process mirros the handling of video format changes, such as when H264 data format changes between AnnexB and AVCC. With this change, `DecoderAgent` simply cues the `PDMFactory` to create a `MediaChangeMonitor` that wraps the underlying decoder, similar to the approach used for video decoders, ensuring correct handling of both AAC and ADTS formats without altering other components. Depends on D220505 Differential Revision: https://phabricator.services.mozilla.com/D220506 UltraBlame original commit: 02c9b1fb72c80c7ad2e73174d2c45e0eead0b26d
diff --git a/dom/media/platforms/PDMFactory.cpp b/dom/media/platforms/PDMFactory.cpp
@@ -409,15 +409,19 @@ PDMFactory::CreateDecoderWithPDM(PlatformDecoderModule* aPDM,
   }
 
   if (config.IsAudio()) {
+    if (MP4Decoder::IsAAC(config.mMimeType) && !aParams.mUseNullDecoder.mUse &&
+        aParams.mWrappers.contains(media::Wrapper::MediaChangeMonitor)) {
+      
+      return MediaChangeMonitor::Create(this, aParams);
+    }
     RefPtr<PlatformDecoderModule::CreateDecoderPromise> p;
     p = aPDM->AsyncCreateDecoder(aParams)->Then(
         GetCurrentSerialEventTarget(), __func__,
         [params = CreateDecoderParamsForAsync(aParams)](
             RefPtr<MediaDataDecoder>&& aDecoder) {
           RefPtr<MediaDataDecoder> decoder = std::move(aDecoder);
-          if (!params.mNoWrapper.mDontUseWrapper) {
-            decoder =
-                new AudioTrimmer(decoder.forget(), CreateDecoderParams(params));
+          if (params.mWrappers.contains(media::Wrapper::AudioTrimmer)) {
+            decoder = new AudioTrimmer(decoder.forget());
           }
           return PlatformDecoderModule::CreateDecoderPromise::CreateAndResolve(
               decoder, __func__);
@@ -444,7 +448,8 @@ PDMFactory::CreateDecoderWithPDM(PlatformDecoderModule* aPDM,
 #endif
        VPXDecoder::IsVPX(config.mMimeType) ||
        MP4Decoder::IsHEVC(config.mMimeType)) &&
-      !aParams.mUseNullDecoder.mUse && !aParams.mNoWrapper.mDontUseWrapper) {
+      !aParams.mUseNullDecoder.mUse &&
+      aParams.mWrappers.contains(media::Wrapper::MediaChangeMonitor)) {
     return MediaChangeMonitor::Create(this, aParams);
   }
   return aPDM->AsyncCreateDecoder(aParams);
diff --git a/dom/media/platforms/PlatformDecoderModule.h b/dom/media/platforms/PlatformDecoderModule.h
@@ -43,22 +43,36 @@ static LazyLogModule sPDMLog("PlatformDecoderModule");
 
 namespace media {
 
-enum class Option {
-  Default,
-  LowLatency,
-  HardwareDecoderNotAllowed,
-  FullH264Parsing,
-  ErrorIfNoInitializationData,  
-                                
-                                
-                                
-  DefaultPlaybackDeviceMono,    
-                                
-  KeepOriginalPts,  
-                    
-
-  SENTINEL  
-};
+template <typename T>
+static nsCString EnumSetToString(const EnumSet<T>& aSet) {
+  nsCString str;
+  for (const auto e : aSet) {
+    if (!str.IsEmpty()) {
+      str.AppendLiteral("|");
+    }
+    str.AppendPrintf("%s", EnumValueToString(e));
+  }
+  if (str.IsEmpty()) {
+    str.AppendLiteral("Empty");
+  }
+  return str;
+}
+
+MOZ_DEFINE_ENUM_CLASS_WITH_TOSTRING(
+    Option,
+    (Default, LowLatency, HardwareDecoderNotAllowed, FullH264Parsing,
+     ErrorIfNoInitializationData,  
+                                   
+                                   
+                                   
+     DefaultPlaybackDeviceMono,  
+                                 
+     KeepOriginalPts,  
+                       
+
+     SENTINEL  
+     ));
+
 using OptionSet = EnumSet<Option>;
 
 struct UseNullDecoder {
@@ -68,11 +82,22 @@ struct UseNullDecoder {
 };
 
 
-struct NoWrapper {
-  NoWrapper() = default;
-  explicit NoWrapper(bool aDontUseWrapper) : mDontUseWrapper(aDontUseWrapper) {}
-  bool mDontUseWrapper = false;
-};
+MOZ_DEFINE_ENUM_CLASS_WITH_BASE_AND_TOSTRING(Wrapper, uint8_t,
+                                             (AudioTrimmer,
+                                              MediaChangeMonitor));
+using WrapperSet = EnumSet<Wrapper>;
+static WrapperSet GetDefaultWrapperSet(const TrackInfo& aInfo) {
+  
+  
+  WrapperSet set;
+  if (aInfo.IsVideo()) {
+    set += Wrapper::MediaChangeMonitor;
+  }
+  if (aInfo.IsAudio()) {
+    set += Wrapper::AudioTrimmer;
+  }
+  return set;
+}
 
 struct VideoFrameRate {
   VideoFrameRate() = default;
@@ -104,7 +129,7 @@ struct CreateDecoderParamsForAsync {
   const RefPtr<layers::KnowsCompositor> mKnowsCompositor;
   const RefPtr<GMPCrashHelper> mCrashHelper;
   const media::UseNullDecoder mUseNullDecoder;
-  const media::NoWrapper mNoWrapper;
+  const media::WrapperSet mWrappers;
   const TrackInfo::TrackType mType = TrackInfo::kUndefinedTrack;
   std::function<MediaEventProducer<TrackInfo::TrackType>*()>
       mOnWaitingForKeyEvent;
@@ -118,14 +143,15 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
   using Option = media::Option;
   using OptionSet = media::OptionSet;
   using UseNullDecoder = media::UseNullDecoder;
-  using NoWrapper = media::NoWrapper;
+  using WrapperSet = media::WrapperSet;
   using VideoFrameRate = media::VideoFrameRate;
   enum class EncryptedCustomIdent : bool {
     False,
     True,
   };
 
-  explicit CreateDecoderParams(const TrackInfo& aConfig) : mConfig(aConfig) {}
+  explicit CreateDecoderParams(const TrackInfo& aConfig)
+      : mConfig(aConfig), mWrappers(media::GetDefaultWrapperSet(aConfig)) {}
   CreateDecoderParams(const CreateDecoderParams& aParams) = default;
 
   MOZ_IMPLICIT CreateDecoderParams(const CreateDecoderParamsForAsync& aParams)
@@ -134,7 +160,7 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
         mKnowsCompositor(aParams.mKnowsCompositor),
         mCrashHelper(aParams.mCrashHelper),
         mUseNullDecoder(aParams.mUseNullDecoder),
-        mNoWrapper(aParams.mNoWrapper),
+        mWrappers(aParams.mWrappers),
         mType(aParams.mType),
         mOnWaitingForKeyEvent(aParams.mOnWaitingForKeyEvent),
         mOptions(aParams.mOptions),
@@ -144,7 +170,7 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
 
   template <typename T1, typename... Ts>
   CreateDecoderParams(const TrackInfo& aConfig, T1&& a1, Ts&&... args)
-      : mConfig(aConfig) {
+      : mConfig(aConfig), mWrappers(media::GetDefaultWrapperSet(aConfig)) {
     Set(std::forward<T1>(a1), std::forward<Ts>(args)...);
   }
 
@@ -164,13 +190,44 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
     return *mConfig.GetAsAudioInfo();
   }
 
+  bool IsVideo() const { return mConfig.IsVideo(); }
+
+  bool IsAudio() const { return mConfig.IsAudio(); }
+
   layers::LayersBackend GetLayersBackend() const {
     if (mKnowsCompositor) {
       return mKnowsCompositor->GetCompositorBackendType();
     }
     return layers::LayersBackend::LAYERS_NONE;
   }
 
+  nsCString ToString() const {
+    nsPrintfCString str("CreateDecoderParams @ %p: ", this);
+    str.AppendPrintf("mConfig = %s", mConfig.ToString().get());
+    str.AppendPrintf(", mImageContainer = %p", mImageContainer);
+    str.AppendPrintf(", mError = %s",
+                     mError ? mError->Description().get() : "null");
+    str.AppendPrintf(", mKnowsCompositor = %p", mKnowsCompositor);
+    str.AppendPrintf(", mCrashHelper = %p", mCrashHelper);
+    str.AppendPrintf(", mUseNullDecoder = %s",
+                     mUseNullDecoder.mUse ? "yes" : "no");
+    str.AppendPrintf(", mWrappers = %s", EnumSetToString(mWrappers).get());
+    str.AppendPrintf(", mType = %d", static_cast<int32_t>(mType));
+    str.AppendPrintf(", mOnWaitingForKeyEvent = %s",
+                     mOnWaitingForKeyEvent ? "yes" : "no");
+    str.AppendPrintf(", mOptions = %s", EnumSetToString(mOptions).get());
+    str.AppendPrintf(", mRate = %f", mRate.mValue);
+    str.AppendPrintf(
+        ", mMediaEngineId = %s",
+        mMediaEngineId ? std::to_string(*mMediaEngineId).c_str() : "None");
+    str.AppendPrintf(", mTrackingId = %s",
+                     mTrackingId ? mTrackingId->ToString().get() : "None");
+    str.AppendPrintf(
+        ", mEncryptedCustomIdent = %s",
+        mEncryptedCustomIdent == EncryptedCustomIdent::True ? "true" : "false");
+    return std::move(str);
+  }
+
   
   
   
@@ -180,7 +237,7 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
   layers::KnowsCompositor* mKnowsCompositor = nullptr;
   GMPCrashHelper* mCrashHelper = nullptr;
   media::UseNullDecoder mUseNullDecoder;
-  media::NoWrapper mNoWrapper;
+  WrapperSet mWrappers;
   TrackInfo::TrackType mType = TrackInfo::kUndefinedTrack;
   std::function<MediaEventProducer<TrackInfo::TrackType>*()>
       mOnWaitingForKeyEvent;
@@ -189,7 +246,7 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
   
   Maybe<uint64_t> mMediaEngineId;
   Maybe<TrackingId> mTrackingId;
-  EncryptedCustomIdent mEncryptedCustomIdent;
+  EncryptedCustomIdent mEncryptedCustomIdent = EncryptedCustomIdent::False;
 
  private:
   void Set(layers::ImageContainer* aImageContainer) {
@@ -200,7 +257,7 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
   void Set(UseNullDecoder aUseNullDecoder) {
     mUseNullDecoder = aUseNullDecoder;
   }
-  void Set(NoWrapper aNoWrapper) { mNoWrapper = aNoWrapper; }
+  void Set(const WrapperSet& aWrappers) { mWrappers = aWrappers; }
   void Set(const OptionSet& aOptions) { mOptions = aOptions; }
   void Set(VideoFrameRate aRate) { mRate = aRate; }
   void Set(layers::KnowsCompositor* aKnowsCompositor) {
@@ -232,7 +289,7 @@ struct MOZ_STACK_CLASS CreateDecoderParams final {
     mKnowsCompositor = aParams.mKnowsCompositor;
     mCrashHelper = aParams.mCrashHelper;
     mUseNullDecoder = aParams.mUseNullDecoder;
-    mNoWrapper = aParams.mNoWrapper;
+    mWrappers = aParams.mWrappers;
     mType = aParams.mType;
     mOnWaitingForKeyEvent = aParams.mOnWaitingForKeyEvent;
     mOptions = aParams.mOptions;
@@ -251,24 +308,25 @@ struct MOZ_STACK_CLASS SupportDecoderParams final {
   using Option = media::Option;
   using OptionSet = media::OptionSet;
   using UseNullDecoder = media::UseNullDecoder;
-  using NoWrapper = media::NoWrapper;
+  using WrapperSet = media::WrapperSet;
   using VideoFrameRate = media::VideoFrameRate;
 
-  explicit SupportDecoderParams(const TrackInfo& aConfig) : mConfig(aConfig) {}
+  explicit SupportDecoderParams(const TrackInfo& aConfig)
+      : mConfig(aConfig), mWrappers(media::GetDefaultWrapperSet(aConfig)) {}
 
   explicit SupportDecoderParams(const CreateDecoderParams& aParams)
       : mConfig(aParams.mConfig),
         mError(aParams.mError),
         mKnowsCompositor(aParams.mKnowsCompositor),
         mUseNullDecoder(aParams.mUseNullDecoder),
-        mNoWrapper(aParams.mNoWrapper),
+        mWrappers(aParams.mWrappers),
         mOptions(aParams.mOptions),
         mRate(aParams.mRate),
         mMediaEngineId(aParams.mMediaEngineId) {}
 
   template <typename T1, typename... Ts>
   SupportDecoderParams(const TrackInfo& aConfig, T1&& a1, Ts&&... args)
-      : mConfig(aConfig) {
+      : mConfig(aConfig), mWrappers(media::GetDefaultWrapperSet(aConfig)) {
     Set(std::forward<T1>(a1), std::forward<Ts>(args)...);
   }
 
@@ -279,7 +337,7 @@ struct MOZ_STACK_CLASS SupportDecoderParams final {
   MediaResult* mError = nullptr;
   RefPtr<layers::KnowsCompositor> mKnowsCompositor;
   UseNullDecoder mUseNullDecoder;
-  NoWrapper mNoWrapper;
+  WrapperSet mWrappers;
   OptionSet mOptions = OptionSet(Option::Default);
   VideoFrameRate mRate;
   Maybe<uint64_t> mMediaEngineId;
@@ -292,7 +350,7 @@ struct MOZ_STACK_CLASS SupportDecoderParams final {
   void Set(media::UseNullDecoder aUseNullDecoder) {
     mUseNullDecoder = aUseNullDecoder;
   }
-  void Set(media::NoWrapper aNoWrapper) { mNoWrapper = aNoWrapper; }
+  void Set(const WrapperSet& aWrappers) { mWrappers = aWrappers; }
   void Set(const media::OptionSet& aOptions) { mOptions = aOptions; }
   void Set(media::VideoFrameRate aRate) { mRate = aRate; }
   void Set(layers::KnowsCompositor* aKnowsCompositor) {
diff --git a/dom/media/platforms/wrappers/MediaChangeMonitor.cpp b/dom/media/platforms/wrappers/MediaChangeMonitor.cpp
@@ -662,7 +662,7 @@ MediaChangeMonitor::MediaChangeMonitor(
     MediaDataDecoder* aDecoder, const CreateDecoderParams& aParams)
     : mChangeMonitor(std::move(aCodecChangeMonitor)),
       mPDMFactory(aPDMFactory),
-      mCurrentConfig(aParams.VideoConfig()),
+      mCurrentConfig(aParams.mConfig.Clone()),
       mDecoder(aDecoder),
       mParams(aParams) {}
 
@@ -671,20 +671,25 @@ RefPtr<PlatformDecoderModule::CreateDecoderPromise> MediaChangeMonitor::Create(
     PDMFactory* aPDMFactory, const CreateDecoderParams& aParams) {
   LOG("MediaChangeMonitor::Create, params = %s", aParams.ToString().get());
   UniquePtr<CodecChangeMonitor> changeMonitor;
-  const VideoInfo& currentConfig = aParams.VideoConfig();
-  if (VPXDecoder::IsVPX(currentConfig.mMimeType)) {
-    changeMonitor = MakeUnique<VPXChangeMonitor>(currentConfig);
+  if (aParams.IsVideo()) {
+    const VideoInfo& config = aParams.VideoConfig();
+    if (VPXDecoder::IsVPX(config.mMimeType)) {
+      changeMonitor = MakeUnique<VPXChangeMonitor>(config);
 #ifdef MOZ_AV1
-  } else if (AOMDecoder::IsAV1(currentConfig.mMimeType)) {
-    changeMonitor = MakeUnique<AV1ChangeMonitor>(currentConfig);
+    } else if (AOMDecoder::IsAV1(config.mMimeType)) {
+      changeMonitor = MakeUnique<AV1ChangeMonitor>(config);
 #endif
-  } else if (MP4Decoder::IsHEVC(currentConfig.mMimeType)) {
-    changeMonitor = MakeUnique<HEVCChangeMonitor>(currentConfig);
+    } else if (MP4Decoder::IsHEVC(config.mMimeType)) {
+      changeMonitor = MakeUnique<HEVCChangeMonitor>(config);
+    } else {
+      MOZ_ASSERT(MP4Decoder::IsH264(config.mMimeType));
+      changeMonitor = MakeUnique<H264ChangeMonitor>(
+          config, aParams.mOptions.contains(
+                      CreateDecoderParams::Option::FullH264Parsing));
+    }
   } else {
-    MOZ_ASSERT(MP4Decoder::IsH264(currentConfig.mMimeType));
-    changeMonitor = MakeUnique<H264ChangeMonitor>(
-        currentConfig, aParams.mOptions.contains(
-                           CreateDecoderParams::Option::FullH264Parsing));
+    MOZ_ASSERT(MP4Decoder::IsAAC(aParams.AudioConfig().mMimeType));
+    changeMonitor = MakeUnique<AACCodecChangeMonitor>(aParams.AudioConfig());
   }
 
   
@@ -920,8 +925,8 @@ void MediaChangeMonitor::SetSeekThreshold(const media::TimeUnit& aTime) {
 
 RefPtr<MediaChangeMonitor::CreateDecoderPromise>
 MediaChangeMonitor::CreateDecoder() {
-  mCurrentConfig = *mChangeMonitor->Config().GetAsVideoInfo();
-  CreateDecoderParams currentParams = {mCurrentConfig, mParams};
+  mCurrentConfig = mChangeMonitor->Config().Clone();
+  CreateDecoderParams currentParams = {*mCurrentConfig, mParams};
   currentParams.mWrappers -= media::Wrapper::MediaChangeMonitor;
   LOG("MediaChangeMonitor::CreateDecoder, current params = %s",
       currentParams.ToString().get());
diff --git a/dom/media/platforms/wrappers/MediaChangeMonitor.h b/dom/media/platforms/wrappers/MediaChangeMonitor.h
@@ -127,7 +127,7 @@ class MediaChangeMonitor final
 
   UniquePtr<CodecChangeMonitor> mChangeMonitor;
   RefPtr<PDMFactory> mPDMFactory;
-  VideoInfo mCurrentConfig;
+  UniquePtr<TrackInfo> mCurrentConfig;
   nsCOMPtr<nsISerialEventTarget> mThread;
   RefPtr<MediaDataDecoder> mDecoder;
   MozPromiseRequestHolder<CreateDecoderPromise> mDecoderRequest;
diff --git a/dom/media/webcodecs/DecoderAgent.cpp b/dom/media/webcodecs/DecoderAgent.cpp
@@ -7,6 +7,7 @@
 #include "DecoderAgent.h"
 
 #include "ImageContainer.h"
+#include "MP4Decoder.h"
 #include "MediaDataDecoderProxy.h"
 #include "PDMFactory.h"
 #include "VideoUtils.h"
@@ -106,6 +107,10 @@ RefPtr<DecoderAgent::ConfigurePromise> DecoderAgent::Configure(
     params.mOptions += CreateDecoderParams::Option::LowLatency;
   }
   
+  if (MP4Decoder::IsAAC(mInfo->mMimeType)) {
+    params.mWrappers += media::Wrapper::MediaChangeMonitor;
+  }
+  
   if (StaticPrefs::media_test_null_decoder_creation_failure()) {
     params.mUseNullDecoder = CreateDecoderParams::UseNullDecoder(true);
   }
@@ -114,10 +119,10 @@ RefPtr<DecoderAgent::ConfigurePromise> DecoderAgent::Configure(
   
   params.mOptions += CreateDecoderParams::Option::KeepOriginalPts;
 
-  LOG("DecoderAgent #%d (%p) is creating a decoder - PreferSW: %s, "
-      "low-latency: %s",
-      mId, this, aPreferSoftwareDecoder ? "yes" : "no",
-      aLowLatency ? "yes" : "no");
+  LOG("DecoderAgent #%d (%p) is creating a decoder (mime: %s) - PreferSW: %s, "
+      "low-latency: %s, create-decoder-params: %s",
+      mId, this, mInfo->mMimeType.get(), aPreferSoftwareDecoder ? "yes" : "no",
+      aLowLatency ? "yes" : "no", params.ToString().get());
 
   RefPtr<ConfigurePromise> p = mConfigurePromise.Ensure(__func__);
 
