From fa8772efb5cf7a314a1bb3f27b77557b0576d2df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Thu, 22 May 2025 12:29:49 +0200 Subject: [PATCH 01/11] Add `onUrlLoaded` callback to `WebViewWidgetMessageInterceptor` This way we can know when the EC page has been loaded and can start interacting with the JS code in it. --- .../android/features/call/impl/ui/CallScreenView.kt | 1 + .../impl/utils/WebViewWidgetMessageInterceptor.kt | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt index aabc3338a5d..a60a3dbc1ec 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt @@ -123,6 +123,7 @@ internal fun CallScreenView( onWebViewCreate = { webView -> val interceptor = WebViewWidgetMessageInterceptor( webView = webView, + onUrlLoaded = {}, onError = { state.eventSink(CallScreenEvents.OnWebViewError(it)) }, ) state.eventSink(CallScreenEvents.SetupMessageChannels(interceptor)) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewWidgetMessageInterceptor.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewWidgetMessageInterceptor.kt index 8fc5d8b7d62..55adc246e9e 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewWidgetMessageInterceptor.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewWidgetMessageInterceptor.kt @@ -26,6 +26,7 @@ import timber.log.Timber class WebViewWidgetMessageInterceptor( private val webView: WebView, + private val onUrlLoaded: (String) -> Unit, private val onError: (String?) -> Unit, ) : WidgetMessageInterceptor { companion object { @@ -44,13 +45,13 @@ class WebViewWidgetMessageInterceptor( .build() webView.webViewClient = object : WebViewClient() { - override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) { + override fun onPageStarted(view: WebView, url: String, favicon: Bitmap?) { super.onPageStarted(view, url, favicon) // Due to https://github.com/element-hq/element-x-android/issues/4097 // we need to supply a logging implementation that correctly includes // objects in log lines. - view?.evaluateJavascript( + view.evaluateJavascript( """ function logFn(consoleLogFn, ...args) { consoleLogFn( @@ -72,7 +73,7 @@ class WebViewWidgetMessageInterceptor( // This listener will receive both messages: // - EC widget API -> Element X (message.data.api == "fromWidget") // - Element X -> EC widget API (message.data.api == "toWidget"), we should ignore these - view?.evaluateJavascript( + view.evaluateJavascript( """ window.addEventListener('message', function(event) { let message = {data: event.data, origin: event.origin} @@ -90,6 +91,10 @@ class WebViewWidgetMessageInterceptor( ) } + override fun onPageFinished(view: WebView, url: String) { + onUrlLoaded(url) + } + override fun onReceivedError(view: WebView?, request: WebResourceRequest?, error: WebResourceError?) { // No network for instance, transmit the error Timber.e("onReceivedError error: ${error?.errorCode} ${error?.description}") From eeb2ed95d1f593c96a9b16b2f29651664076ef4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Thu, 22 May 2025 12:53:46 +0200 Subject: [PATCH 02/11] Add `WebViewAudioManager` component and use it instead of the `AudioManager` extension functions This component is responsible of tracking the audio device usage on both the native OS and the webview and coordinating them. --- .../features/call/impl/ui/CallScreenView.kt | 62 +-- .../call/impl/utils/WebViewAudioManager.kt | 413 ++++++++++++++++++ .../androidutils/compat/AudioManager.kt | 70 --- 3 files changed, 433 insertions(+), 112 deletions(-) create mode 100644 features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt index a60a3dbc1ec..c0997c617eb 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt @@ -8,10 +8,6 @@ package io.element.android.features.call.impl.ui import android.annotation.SuppressLint -import android.content.Context -import android.media.AudioDeviceCallback -import android.media.AudioDeviceInfo -import android.media.AudioManager import android.util.Log import android.view.ViewGroup import android.webkit.ConsoleMessage @@ -35,17 +31,15 @@ import androidx.compose.ui.platform.LocalInspectionMode import androidx.compose.ui.res.stringResource import androidx.compose.ui.tooling.preview.PreviewParameter import androidx.compose.ui.viewinterop.AndroidView -import androidx.core.content.getSystemService import io.element.android.compound.tokens.generated.CompoundIcons import io.element.android.features.call.impl.R import io.element.android.features.call.impl.pip.PictureInPictureEvents import io.element.android.features.call.impl.pip.PictureInPictureState import io.element.android.features.call.impl.pip.PictureInPictureStateProvider import io.element.android.features.call.impl.pip.aPictureInPictureState +import io.element.android.features.call.impl.utils.WebViewAudioManager import io.element.android.features.call.impl.utils.WebViewPipController import io.element.android.features.call.impl.utils.WebViewWidgetMessageInterceptor -import io.element.android.libraries.androidutils.compat.disableExternalAudioDevice -import io.element.android.libraries.androidutils.compat.enableExternalAudioDevice import io.element.android.libraries.architecture.AsyncData import io.element.android.libraries.designsystem.components.ProgressDialog import io.element.android.libraries.designsystem.components.button.BackButton @@ -108,6 +102,7 @@ internal fun CallScreenView( onSubmit = { state.eventSink(CallScreenEvents.Hangup) }, ) } else { + var webViewAudioManager by remember { mutableStateOf(null) } CallWebView( modifier = Modifier .padding(padding) @@ -120,15 +115,27 @@ internal fun CallScreenView( val callback: RequestPermissionCallback = { request.grant(it) } requestPermissions(androidPermissions.toTypedArray(), callback) }, - onWebViewCreate = { webView -> + onCreateWebView = { webView -> val interceptor = WebViewWidgetMessageInterceptor( webView = webView, - onUrlLoaded = {}, + onUrlLoaded = { url -> + if (webViewAudioManager?.isInCallMode?.get() == false) { + Timber.d("URL $url is loaded, starting in-call audio mode") + webViewAudioManager?.onCallStarted() + } else { + Timber.d("Can't start in-call audio mode since the app is already in it.") + } + }, onError = { state.eventSink(CallScreenEvents.OnWebViewError(it)) }, ) + webViewAudioManager = WebViewAudioManager(webView) state.eventSink(CallScreenEvents.SetupMessageChannels(interceptor)) val pipController = WebViewPipController(webView) pipState.eventSink(PictureInPictureEvents.SetPipController(pipController)) + }, + onDestroyWebView = { + // Reset audio mode + webViewAudioManager?.onCallStopped() } ) when (state.urlState) { @@ -153,7 +160,8 @@ private fun CallWebView( url: AsyncData, userAgent: String, onPermissionsRequest: (PermissionRequest) -> Unit, - onWebViewCreate: (WebView) -> Unit, + onCreateWebView: (WebView) -> Unit, + onDestroyWebView: (WebView) -> Unit, modifier: Modifier = Modifier, ) { if (LocalInspectionMode.current) { @@ -161,13 +169,11 @@ private fun CallWebView( Text("WebView - can't be previewed") } } else { - var audioDeviceCallback: AudioDeviceCallback? by remember { mutableStateOf(null) } AndroidView( modifier = modifier, factory = { context -> - audioDeviceCallback = context.setupAudioConfiguration() WebView(context).apply { - onWebViewCreate(this) + onCreateWebView(this) setup(userAgent, onPermissionsRequest) } }, @@ -177,41 +183,13 @@ private fun CallWebView( } }, onRelease = { webView -> - // Reset audio mode - webView.context.releaseAudioConfiguration(audioDeviceCallback) + onDestroyWebView(webView) webView.destroy() } ) } } -private fun Context.setupAudioConfiguration(): AudioDeviceCallback? { - val audioManager = getSystemService() ?: return null - // Set 'voice call' mode so volume keys actually control the call volume - audioManager.mode = AudioManager.MODE_IN_COMMUNICATION - audioManager.enableExternalAudioDevice() - return object : AudioDeviceCallback() { - override fun onAudioDevicesAdded(addedDevices: Array?) { - Timber.d("Audio devices added") - audioManager.enableExternalAudioDevice() - } - - override fun onAudioDevicesRemoved(removedDevices: Array?) { - Timber.d("Audio devices removed") - audioManager.enableExternalAudioDevice() - } - }.also { - audioManager.registerAudioDeviceCallback(it, null) - } -} - -private fun Context.releaseAudioConfiguration(audioDeviceCallback: AudioDeviceCallback?) { - val audioManager = getSystemService() ?: return - audioManager.unregisterAudioDeviceCallback(audioDeviceCallback) - audioManager.disableExternalAudioDevice() - audioManager.mode = AudioManager.MODE_NORMAL -} - @SuppressLint("SetJavaScriptEnabled") private fun WebView.setup( userAgent: String, diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt new file mode 100644 index 00000000000..9ff0ce148db --- /dev/null +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt @@ -0,0 +1,413 @@ +/* + * Copyright 2025 New Vector Ltd. + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial + * Please see LICENSE files in the repository root for full details. + */ + +package io.element.android.features.call.impl.utils + +import android.content.Context +import android.media.AudioDeviceCallback +import android.media.AudioDeviceInfo +import android.media.AudioManager +import android.os.Build +import android.os.PowerManager +import android.webkit.JavascriptInterface +import android.webkit.WebView +import androidx.core.content.getSystemService +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.launch +import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import kotlinx.serialization.json.Json +import timber.log.Timber +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicBoolean + +/** + * This class manages the audio devices for a WebView. + * + * It listens for audio device changes and updates the WebView with the available devices. + * It also handles the selection of the audio device by the user in the WebView and the default audio device based on the device type. + */ +class WebViewAudioManager( + private val webView: WebView, +) { + // The list of device types that are considered as communication devices, sorted by likelihood of it being used for communication. + private val wantedDeviceTypes = listOf( + // Paired bluetooth device with microphone + AudioDeviceInfo.TYPE_BLUETOOTH_SCO, + // USB devices which can play or record audio + AudioDeviceInfo.TYPE_USB_HEADSET, + AudioDeviceInfo.TYPE_USB_DEVICE, + AudioDeviceInfo.TYPE_USB_ACCESSORY, + // Wired audio devices + AudioDeviceInfo.TYPE_WIRED_HEADSET, + AudioDeviceInfo.TYPE_WIRED_HEADPHONES, + // The built-in speaker of the device + AudioDeviceInfo.TYPE_BUILTIN_SPEAKER, + // The built-in earpiece of the device + AudioDeviceInfo.TYPE_BUILTIN_EARPIECE, + ) + + private val audioManager = webView.context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + private val proximitySensorWakeLock by lazy { + webView.context.getSystemService() + ?.takeIf { it.isWakeLockLevelSupported(PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK) } + ?.newWakeLock(PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, "${webView.context.packageName}:ProximitySensorCallWakeLock") + } + + /** + * This listener tracks the current communication device and updates the WebView when it changes. + */ + private val commsDeviceChangedListener = AudioManager.OnCommunicationDeviceChangedListener { device -> + if (device?.id == expectedNewCommunicationDeviceId) { + if (device != null) { + expectedNewCommunicationDeviceId = null + Timber.d("Audio device changed, type: ${device.type}") + updateSelectedAudioDeviceInWebView(device.id.toString()) + } else { + Timber.d("No audio device selected") + } + } else { + // We were expecting a device change but it didn't happen, so we should retry + val expectedDeviceId = expectedNewCommunicationDeviceId + if (expectedDeviceId != null) { + // Remove the expected id so we only retry once + expectedNewCommunicationDeviceId = null + audioManager.selectAudioDevice(expectedDeviceId.toString()) + } + } + } + + /** + * This callback is used to listen for audio device changes coming from the OS. + */ + private val audioDeviceCallback = object : AudioDeviceCallback() { + override fun onAudioDevicesAdded(addedDevices: Array?) { + val validNewDevices = addedDevices.orEmpty().filter { it.type in wantedDeviceTypes && it.isSink } + if (validNewDevices.isEmpty()) return + + // We need to calculate the available devices ourselves, since calling `listAudioDevices` will return an outdated list + val audioDevices = (listAudioDevices() + validNewDevices).distinctBy { it.id } + setAvailableAudioDevices(audioDevices.map(SerializableAudioDevice::fromAudioDeviceInfo)) + // This should automatically switch to a new device if it has a higher priority than the current one + selectDefaultAudioDevice(audioDevices) + } + + override fun onAudioDevicesRemoved(removedDevices: Array?) { + // Update the available devices + setAvailableAudioDevices() + + // Unless the removed device is the current one, we don't need to do anything else + val removedCurrentDevice = removedDevices.orEmpty().any { it.id == currentDeviceId } + if (!removedCurrentDevice) return + + val previousDevice = previousSelectedDevice + if (previousDevice != null) { + previousSelectedDevice = null + // If we have a previous device, we should select it again + audioManager.selectAudioDevice(previousDevice.id.toString()) + } else { + // If we don't have a previous device, we should select the default one + selectDefaultAudioDevice() + } + } + } + + /** + * The currently used audio device id. + */ + private var currentDeviceId: Int? = null + + /** + * When a new audio device is selected but not yet set as the communication device by the OS, this id is used to check if the device is the expected one. + */ + private var expectedNewCommunicationDeviceId: Int? = null + + /** + * Previously selected device, used to restore the selection when the selected device is removed. + */ + private var previousSelectedDevice: AudioDeviceInfo? = null + + /** + * Marks if the WebView audio is in call mode or not. + */ + val isInCallMode = AtomicBoolean(false) + + init { + // Apparently, registering the javascript interface takes a while, so registering and immediately using it doesn't work + // We register it ahead of time to avoid this issue + registerWebViewDeviceSelectedCallback() + } + + /** + * Call this method when the call starts to enable in-call audio mode. + * + * It'll set the audio mode to [AudioManager.MODE_IN_COMMUNICATION] if possible, register the audio device callback and set the available audio devices. + */ + fun onCallStarted() { + if (!isInCallMode.compareAndSet(false, true)) { + Timber.w("Audio: tried to enable webview in-call audio mode while already in it") + return + } + + Timber.d("Audio: enabling webview in-call audio mode") + + // TODO double check used audio stream + audioManager.mode = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + // Set 'voice call' mode so volume keys actually control the call volume + AudioManager.MODE_IN_COMMUNICATION + } else { + // Workaround for Android 12 and lower, otherwise changing the audio device doesn't work + AudioManager.MODE_NORMAL + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + audioManager.addOnCommunicationDeviceChangedListener(Executors.newSingleThreadExecutor(), commsDeviceChangedListener) + } + + audioManager.registerAudioDeviceCallback(audioDeviceCallback, null) + + setAvailableAudioDevices() + selectDefaultAudioDevice() + setWebViewOnAudioDeviceSelectedCallback() + } + + /** + * Call this method when the call stops to disable in-call audio mode. + * + * It's the counterpart of [onCallStarted], and should be called as a pair with it once the call has ended. + */ + fun onCallStopped() { + if (!isInCallMode.compareAndSet(true, false)) { + Timber.w("Audio: tried to disable webview in-call audio mode while already disabled") + return + } + audioManager.unregisterAudioDeviceCallback(audioDeviceCallback) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + audioManager.removeOnCommunicationDeviceChangedListener(commsDeviceChangedListener) + } + + if (proximitySensorWakeLock?.isHeld == true) { + proximitySensorWakeLock?.release() + } + + audioManager.mode = AudioManager.MODE_NORMAL + } + + /** + * Registers the WebView audio device selected callback. + * + * This should be called when the WebView is created to ensure that the callback is set before any audio device selection is made. + */ + private fun registerWebViewDeviceSelectedCallback() { + val webViewAudioDeviceSelectedCallback = WebViewAudioOutputCallback { selectedDeviceId -> + Timber.d("Audio device selected in webview, id: $selectedDeviceId") + previousSelectedDevice = listAudioDevices().find { it.id.toString() == selectedDeviceId } + audioManager.selectAudioDevice(selectedDeviceId) + } + Timber.d("Setting onAudioDeviceSelectedCallback javascript interface in webview") + webView.addJavascriptInterface(webViewAudioDeviceSelectedCallback, "onAudioDeviceSelectedCallback") + } + + /** + * Assigns the callback in the WebView to be called when the user selects an audio device. + * + * It should be called with some delay after [registerWebViewDeviceSelectedCallback]. + */ + private fun setWebViewOnAudioDeviceSelectedCallback() { + Timber.d("Adding callback in controls.onOutputDeviceSelect") + webView.evaluateJavascript("controls.onOutputDeviceSelect = (id) => { onAudioDeviceSelectedCallback.setOutputDevice(id); };", null) + } + + /** + * Returns the list of available audio devices. + * + * On Android 11 ([Build.VERSION_CODES.R]) and lower, it returns the list of output devices as a fallback. + */ + private fun listAudioDevices(): List { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + audioManager.availableCommunicationDevices + } else { + val rawAudioDevices = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS) + rawAudioDevices.filter { it.type in wantedDeviceTypes && it.isSink } + } + } + + /** + * Sets the available audio devices in the WebView. + * + * @param devices The list of audio devices to set. If not provided, it will use the current list of audio devices. + */ + private fun setAvailableAudioDevices( + devices: List = listAudioDevices().map(SerializableAudioDevice::fromAudioDeviceInfo), + ) { + Timber.d("Updating available audio devices") + val jsonSerializer = Json { + encodeDefaults = true + explicitNulls = false + } + val deviceList = jsonSerializer.encodeToString(devices) + webView.evaluateJavascript("controls.setAvailableOutputDevices($deviceList);", { + Timber.d("Audio: setAvailableOutputDevices result: $it") + }) + } + + /** + * Selects the default audio device based on the available devices. + * + * @param availableDevices The list of available audio devices to select from. If not provided, it will use the current list of audio devices. + */ + private fun selectDefaultAudioDevice(availableDevices: List = listAudioDevices()) { + val selectedDevice = availableDevices.minByOrNull { + wantedDeviceTypes.indexOf(it.type).let { index -> + // If the device type is not in the wantedDeviceTypes list, we give it a low priority + if (index == -1) Int.MAX_VALUE else index + } + } + + expectedNewCommunicationDeviceId = selectedDevice?.id + audioManager.selectAudioDevice(selectedDevice) + + selectedDevice?.let { + updateSelectedAudioDeviceInWebView(it.id.toString()) + } ?: run { + Timber.w("Audio: unable to select default audio device") + } + } + + /** + * Updates the WebView's UI to reflect the selected audio device. + * + * @param deviceId The id of the selected audio device. + */ + private fun updateSelectedAudioDeviceInWebView(deviceId: String) { + MainScope().launch { webView.evaluateJavascript("controls.setOutputDevice('$deviceId');", null) } + } + + /** + * Selects the audio device on the OS based on the provided device id. + * + * It will select the device only if it is available in the list of audio devices. + * + * @param device The id of the audio device to select. + */ + private fun AudioManager.selectAudioDevice(device: String) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + val audioDevice = availableCommunicationDevices.find { it.id.toString() == device } + selectAudioDevice(audioDevice) + } else { + val rawAudioDevices = getDevices(AudioManager.GET_DEVICES_OUTPUTS) + val audioDevice = rawAudioDevices.find { it.id.toString() == device } + selectAudioDevice(audioDevice) + } + } + + /** + * Selects the audio device on the OS based on the provided device info. + * + * @param device The info of the audio device to select, or none to clear the selected device. + */ + @Suppress("DEPRECATION") + private fun AudioManager.selectAudioDevice(device: AudioDeviceInfo?) { + currentDeviceId = device?.id + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + if (device != null) { + if (device != communicationDevice) { + setCommunicationDevice(device) + } + } else { + audioManager.clearCommunicationDevice() + } + } else { + // On Android 11 and lower, we don't have the concept of communication devices + // We have to call the right methods based on the device type + if (device != null) { + isSpeakerphoneOn = device.type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER + isBluetoothScoOn = device.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO + } else { + isSpeakerphoneOn = false + isBluetoothScoOn = false + } + } + + @Suppress("WakeLock", "WakeLockTimeout") + if (device?.type == AudioDeviceInfo.TYPE_BUILTIN_EARPIECE && proximitySensorWakeLock?.isHeld == false) { + // If the device is the built-in earpiece, we need to acquire the proximity sensor wake lock + proximitySensorWakeLock?.acquire() + } else if (proximitySensorWakeLock?.isHeld == true) { + // If the device is no longer the earpiece, we need to release the wake lock + proximitySensorWakeLock?.release() + } + } +} + +/** + * This class is used to handle the audio device selection in the WebView. + * It listens for the audio device selection event and calls the callback with the selected device ID. + */ +private class WebViewAudioOutputCallback( + private val callback: (String) -> Unit, +) { + @JavascriptInterface + fun setOutputDevice(id: String) { + Timber.d("Audio device selected in webview, id: $id") + callback(id) + } +} + +private fun deviceName(type: Int, name: String): String { + // TODO maybe translate these? + val typePart = when (type) { + AudioDeviceInfo.TYPE_BLUETOOTH_SCO -> "Bluetooth" + AudioDeviceInfo.TYPE_USB_ACCESSORY -> "USB accessory" + AudioDeviceInfo.TYPE_USB_DEVICE -> "USB device" + AudioDeviceInfo.TYPE_USB_HEADSET -> "USB headset" + AudioDeviceInfo.TYPE_WIRED_HEADSET -> "Wired headset" + AudioDeviceInfo.TYPE_WIRED_HEADPHONES -> "Wired headphones" + AudioDeviceInfo.TYPE_BUILTIN_SPEAKER -> "Built-in speaker" + AudioDeviceInfo.TYPE_BUILTIN_EARPIECE -> "Built-in earpiece" + else -> "Unknown:" + } + return if (isBuiltIn(type)) { + typePart + } else { + "$typePart - $name" + } +} + +private fun isBuiltIn(type: Int): Boolean = when (type) { + AudioDeviceInfo.TYPE_BUILTIN_SPEAKER, + AudioDeviceInfo.TYPE_BUILTIN_EARPIECE, + AudioDeviceInfo.TYPE_BUILTIN_MIC, + AudioDeviceInfo.TYPE_BUILTIN_SPEAKER_SAFE -> true + else -> false +} + +/** + * This class is used to serialize the audio device information to JSON. + */ +@Suppress("unused") +@Serializable +internal data class SerializableAudioDevice( + val id: String, + val name: String, + @Transient val type: Int = 0, + // These have to be part of the constructor for the JSON serializer to pick them up + val isEarpiece: Boolean = type == AudioDeviceInfo.TYPE_BUILTIN_EARPIECE, + val isSpeaker: Boolean = type == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER, + val isExternalHeadset: Boolean = type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO, +) { + companion object { + fun fromAudioDeviceInfo(audioDeviceInfo: AudioDeviceInfo): SerializableAudioDevice { + return SerializableAudioDevice( + id = audioDeviceInfo.id.toString(), + name = deviceName(type = audioDeviceInfo.type, name = audioDeviceInfo.productName.toString()), + type = audioDeviceInfo.type, + ) + } + } +} diff --git a/libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/compat/AudioManager.kt b/libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/compat/AudioManager.kt index 780cce94612..e69de29bb2d 100644 --- a/libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/compat/AudioManager.kt +++ b/libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/compat/AudioManager.kt @@ -1,70 +0,0 @@ -/* - * Copyright 2025 New Vector Ltd. - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial - * Please see LICENSE files in the repository root for full details. - */ - -package io.element.android.libraries.androidutils.compat - -import android.media.AudioDeviceInfo -import android.media.AudioManager -import android.os.Build -import io.element.android.libraries.core.data.tryOrNull -import timber.log.Timber - -fun AudioManager.enableExternalAudioDevice() { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - // The list of device types that are considered as communication devices, sorted by likelihood of it being used for communication. - val wantedDeviceTypes = listOf( - // Paired bluetooth device with microphone - AudioDeviceInfo.TYPE_BLUETOOTH_SCO, - // USB devices which can play or record audio - AudioDeviceInfo.TYPE_USB_HEADSET, - AudioDeviceInfo.TYPE_USB_DEVICE, - AudioDeviceInfo.TYPE_USB_ACCESSORY, - // Wired audio devices - AudioDeviceInfo.TYPE_WIRED_HEADSET, - AudioDeviceInfo.TYPE_WIRED_HEADPHONES, - // The built-in speaker of the device - AudioDeviceInfo.TYPE_BUILTIN_SPEAKER, - // The built-in earpiece of the device - AudioDeviceInfo.TYPE_BUILTIN_EARPIECE, - ) - val devices = availableCommunicationDevices - val selectedDevice = devices.minByOrNull { - wantedDeviceTypes.indexOf(it.type).let { index -> - // If the device type is not in the wantedDeviceTypes list, we give it a low priority - if (index == -1) Int.MAX_VALUE else index - } - } - selectedDevice?.let { device -> - Timber.d("Audio device selected, type: ${device.type}") - tryOrNull( - onException = { failure -> - Timber.e(failure, "Audio: exception when setting communication device") - } - ) { - setCommunicationDevice(device).also { - if (!it) { - Timber.w("Audio: unable to set the communication device") - } - } - } - } - } else { - // If we don't have access to the new APIs, use the deprecated ones - @Suppress("DEPRECATION") - isSpeakerphoneOn = true - } -} - -fun AudioManager.disableExternalAudioDevice() { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - clearCommunicationDevice() - } else { - // If we don't have access to the new APIs, use the deprecated ones - @Suppress("DEPRECATION") - isSpeakerphoneOn = false - } -} From dece44733f89ebddbb221824fec6ebc4bbecb8e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Thu, 22 May 2025 12:54:31 +0200 Subject: [PATCH 03/11] Enable controlling the audio devices in Element Call from the OS instead of automatically detecting them --- .../matrix/impl/widget/DefaultCallWidgetSettingsProvider.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/widget/DefaultCallWidgetSettingsProvider.kt b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/widget/DefaultCallWidgetSettingsProvider.kt index 77152776aa4..eed81c2dcbd 100644 --- a/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/widget/DefaultCallWidgetSettingsProvider.kt +++ b/libraries/matrix/impl/src/main/kotlin/io/element/android/libraries/matrix/impl/widget/DefaultCallWidgetSettingsProvider.kt @@ -49,7 +49,7 @@ class DefaultCallWidgetSettingsProvider @Inject constructor( sentryEnvironment = if (buildMeta.buildType == BuildType.RELEASE) "RELEASE" else "DEBUG", parentUrl = null, hideHeader = true, - controlledMediaDevices = false, + controlledMediaDevices = true, ) val rustWidgetSettings = newVirtualElementCallWidget(options) return MatrixWidgetSettings.fromRustWidgetSettings(rustWidgetSettings) From 20e008de717dcae3c4644165f290b5e712459dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Thu, 22 May 2025 12:54:46 +0200 Subject: [PATCH 04/11] Simplify the window flags in `ElementCallActivity` --- .../features/call/impl/ui/ElementCallActivity.kt | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/ElementCallActivity.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/ElementCallActivity.kt index cec8a81c588..870c2244648 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/ElementCallActivity.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/ElementCallActivity.kt @@ -81,13 +81,14 @@ class ElementCallActivity : applicationContext.bindings().inject(this) - @Suppress("DEPRECATION") - window.addFlags( - WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON or - WindowManager.LayoutParams.FLAG_ALLOW_LOCK_WHILE_SCREEN_ON or - WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED or - WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON - ) + window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O_MR1) { + setShowWhenLocked(true) + } else { + @Suppress("DEPRECATION") + window.addFlags(WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED) + } setCallType(intent) // If presenter is not created at this point, it means we have no call to display, the Activity is finishing, so return early From 3b924aeb7311ea0e13daf4d1d231a8452b24e3a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Tue, 3 Jun 2025 11:33:39 +0200 Subject: [PATCH 05/11] Try working around the issue where the default audio device wasn't using the right audio stream --- .../call/impl/utils/WebViewAudioManager.kt | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt index 9ff0ce148db..17e3cd9e132 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt @@ -17,6 +17,7 @@ import android.webkit.JavascriptInterface import android.webkit.WebView import androidx.core.content.getSystemService import kotlinx.coroutines.MainScope +import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.serialization.Serializable import kotlinx.serialization.Transient @@ -24,6 +25,7 @@ import kotlinx.serialization.json.Json import timber.log.Timber import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicBoolean +import kotlin.time.Duration.Companion.seconds /** * This class manages the audio devices for a WebView. @@ -63,15 +65,11 @@ class WebViewAudioManager( * This listener tracks the current communication device and updates the WebView when it changes. */ private val commsDeviceChangedListener = AudioManager.OnCommunicationDeviceChangedListener { device -> - if (device?.id == expectedNewCommunicationDeviceId) { - if (device != null) { - expectedNewCommunicationDeviceId = null - Timber.d("Audio device changed, type: ${device.type}") - updateSelectedAudioDeviceInWebView(device.id.toString()) - } else { - Timber.d("No audio device selected") - } - } else { + if (device != null && device.id == expectedNewCommunicationDeviceId) { + expectedNewCommunicationDeviceId = null + Timber.d("Audio device changed, type: ${device.type}") + updateSelectedAudioDeviceInWebView(device.id.toString()) + } else if (device != null && device.id != expectedNewCommunicationDeviceId) { // We were expecting a device change but it didn't happen, so we should retry val expectedDeviceId = expectedNewCommunicationDeviceId if (expectedDeviceId != null) { @@ -79,6 +77,10 @@ class WebViewAudioManager( expectedNewCommunicationDeviceId = null audioManager.selectAudioDevice(expectedDeviceId.toString()) } + } else { + Timber.d("Audio device cleared") + expectedNewCommunicationDeviceId = null + audioManager.selectAudioDevice(null) } } @@ -156,7 +158,6 @@ class WebViewAudioManager( Timber.d("Audio: enabling webview in-call audio mode") - // TODO double check used audio stream audioManager.mode = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { // Set 'voice call' mode so volume keys actually control the call volume AudioManager.MODE_IN_COMMUNICATION @@ -165,14 +166,22 @@ class WebViewAudioManager( AudioManager.MODE_NORMAL } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - audioManager.addOnCommunicationDeviceChangedListener(Executors.newSingleThreadExecutor(), commsDeviceChangedListener) - } + MainScope().launch { + // If we register the audio device callback immediately, the webview doesn't seem to be ready + // and we end up with the wrong audio stream being used for controlling the volume (media instead of call) + // 3 seconds is a magic number that seems to work well, but it might need to be adjusted + // Ideally we'd have a callback for this 'audio stream ready' event, but it doesn't exist yet + delay(3.seconds) - audioManager.registerAudioDeviceCallback(audioDeviceCallback, null) + // Registering the audio devices changed callback will also set the default audio device + audioManager.registerAudioDeviceCallback(audioDeviceCallback, null) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + audioManager.addOnCommunicationDeviceChangedListener(Executors.newSingleThreadExecutor(), commsDeviceChangedListener) + } + } setAvailableAudioDevices() - selectDefaultAudioDevice() setWebViewOnAudioDeviceSelectedCallback() } @@ -186,11 +195,13 @@ class WebViewAudioManager( Timber.w("Audio: tried to disable webview in-call audio mode while already disabled") return } - audioManager.unregisterAudioDeviceCallback(audioDeviceCallback) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { audioManager.removeOnCommunicationDeviceChangedListener(commsDeviceChangedListener) + audioManager.clearCommunicationDevice() } + audioManager.unregisterAudioDeviceCallback(audioDeviceCallback) + if (proximitySensorWakeLock?.isHeld == true) { proximitySensorWakeLock?.release() } @@ -316,9 +327,8 @@ class WebViewAudioManager( currentDeviceId = device?.id if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { if (device != null) { - if (device != communicationDevice) { + Timber.d("Setting communication device: ${device.id} - ${deviceName(device.type, device.productName.toString())}") setCommunicationDevice(device) - } } else { audioManager.clearCommunicationDevice() } @@ -334,6 +344,8 @@ class WebViewAudioManager( } } + expectedNewCommunicationDeviceId = null + @Suppress("WakeLock", "WakeLockTimeout") if (device?.type == AudioDeviceInfo.TYPE_BUILTIN_EARPIECE && proximitySensorWakeLock?.isHeld == false) { // If the device is the built-in earpiece, we need to acquire the proximity sensor wake lock From dcccd582e3cb1f82a3ee41b146df7e4333377b54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Tue, 3 Jun 2025 11:37:00 +0200 Subject: [PATCH 06/11] Follow review comments --- .../android/features/call/impl/utils/WebViewAudioManager.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt index 17e3cd9e132..fa227b5750d 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt @@ -32,6 +32,8 @@ import kotlin.time.Duration.Companion.seconds * * It listens for audio device changes and updates the WebView with the available devices. * It also handles the selection of the audio device by the user in the WebView and the default audio device based on the device type. + * + * See also: [Element Call controls docs.](https://github.com/element-hq/element-call/blob/livekit/docs/controls.md#audio-devices) */ class WebViewAudioManager( private val webView: WebView, @@ -382,7 +384,7 @@ private fun deviceName(type: Int, name: String): String { AudioDeviceInfo.TYPE_WIRED_HEADPHONES -> "Wired headphones" AudioDeviceInfo.TYPE_BUILTIN_SPEAKER -> "Built-in speaker" AudioDeviceInfo.TYPE_BUILTIN_EARPIECE -> "Built-in earpiece" - else -> "Unknown:" + else -> "Unknown" } return if (isBuiltIn(type)) { typePart From 56f1fea2e24ee89eb73d27d5841aa6b0ddeca92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Tue, 3 Jun 2025 14:31:22 +0200 Subject: [PATCH 07/11] Add `onAudioTrackReady`, use it to start the audio-device related logic Otherwise, the default audio device won't use the right audio stream. --- .../call/impl/utils/WebViewAudioManager.kt | 93 +++++++++++-------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt index fa227b5750d..18344b37e1d 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt @@ -17,7 +17,6 @@ import android.webkit.JavascriptInterface import android.webkit.WebView import androidx.core.content.getSystemService import kotlinx.coroutines.MainScope -import kotlinx.coroutines.delay import kotlinx.coroutines.launch import kotlinx.serialization.Serializable import kotlinx.serialization.Transient @@ -25,7 +24,6 @@ import kotlinx.serialization.json.Json import timber.log.Timber import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicBoolean -import kotlin.time.Duration.Companion.seconds /** * This class manages the audio devices for a WebView. @@ -136,6 +134,8 @@ class WebViewAudioManager( */ private var previousSelectedDevice: AudioDeviceInfo? = null + private var hasRegisteredCallbacks = false + /** * Marks if the WebView audio is in call mode or not. */ @@ -168,23 +168,7 @@ class WebViewAudioManager( AudioManager.MODE_NORMAL } - MainScope().launch { - // If we register the audio device callback immediately, the webview doesn't seem to be ready - // and we end up with the wrong audio stream being used for controlling the volume (media instead of call) - // 3 seconds is a magic number that seems to work well, but it might need to be adjusted - // Ideally we'd have a callback for this 'audio stream ready' event, but it doesn't exist yet - delay(3.seconds) - - // Registering the audio devices changed callback will also set the default audio device - audioManager.registerAudioDeviceCallback(audioDeviceCallback, null) - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - audioManager.addOnCommunicationDeviceChangedListener(Executors.newSingleThreadExecutor(), commsDeviceChangedListener) - } - } - - setAvailableAudioDevices() - setWebViewOnAudioDeviceSelectedCallback() + setWebViewAndroidNativeBridge() } /** @@ -197,18 +181,24 @@ class WebViewAudioManager( Timber.w("Audio: tried to disable webview in-call audio mode while already disabled") return } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { - audioManager.removeOnCommunicationDeviceChangedListener(commsDeviceChangedListener) - audioManager.clearCommunicationDevice() - } - - audioManager.unregisterAudioDeviceCallback(audioDeviceCallback) if (proximitySensorWakeLock?.isHeld == true) { proximitySensorWakeLock?.release() } audioManager.mode = AudioManager.MODE_NORMAL + + if (!hasRegisteredCallbacks) { + Timber.w("Audio: tried to disable webview in-call audio mode without registering callbacks") + return + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + audioManager.clearCommunicationDevice() + audioManager.removeOnCommunicationDeviceChangedListener(commsDeviceChangedListener) + } + + audioManager.unregisterAudioDeviceCallback(audioDeviceCallback) } /** @@ -217,13 +207,30 @@ class WebViewAudioManager( * This should be called when the WebView is created to ensure that the callback is set before any audio device selection is made. */ private fun registerWebViewDeviceSelectedCallback() { - val webViewAudioDeviceSelectedCallback = WebViewAudioOutputCallback { selectedDeviceId -> - Timber.d("Audio device selected in webview, id: $selectedDeviceId") - previousSelectedDevice = listAudioDevices().find { it.id.toString() == selectedDeviceId } - audioManager.selectAudioDevice(selectedDeviceId) - } - Timber.d("Setting onAudioDeviceSelectedCallback javascript interface in webview") - webView.addJavascriptInterface(webViewAudioDeviceSelectedCallback, "onAudioDeviceSelectedCallback") + val webViewAudioDeviceSelectedCallback = AndroidWebViewAudioBridge( + onAudioDeviceSelected = { selectedDeviceId -> + Timber.d("Audio device selected in webview, id: $selectedDeviceId") + previousSelectedDevice = listAudioDevices().find { it.id.toString() == selectedDeviceId } + audioManager.selectAudioDevice(selectedDeviceId) + }, + onAudioTrackReady = { + MainScope().launch { + // Calling this ahead of time makes the default audio device to not use the right audio stream + setAvailableAudioDevices() + + // Registering the audio devices changed callback will also set the default audio device + audioManager.registerAudioDeviceCallback(audioDeviceCallback, null) + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + audioManager.addOnCommunicationDeviceChangedListener(Executors.newSingleThreadExecutor(), commsDeviceChangedListener) + } + + hasRegisteredCallbacks = true + } + } + ) + Timber.d("Setting androidNativeBridge javascript interface in webview") + webView.addJavascriptInterface(webViewAudioDeviceSelectedCallback, "androidNativeBridge") } /** @@ -231,9 +238,11 @@ class WebViewAudioManager( * * It should be called with some delay after [registerWebViewDeviceSelectedCallback]. */ - private fun setWebViewOnAudioDeviceSelectedCallback() { + private fun setWebViewAndroidNativeBridge() { + Timber.d("Adding callback in controls.onAudioTrackReady") + webView.evaluateJavascript("controls.onAudioTrackReady = () => { console.log('SET TRACK READY SET'); androidNativeBridge.onTrackReady(); };", null) Timber.d("Adding callback in controls.onOutputDeviceSelect") - webView.evaluateJavascript("controls.onOutputDeviceSelect = (id) => { onAudioDeviceSelectedCallback.setOutputDevice(id); };", null) + webView.evaluateJavascript("controls.onOutputDeviceSelect = (id) => { androidNativeBridge.setOutputDevice(id); };", null) } /** @@ -363,13 +372,23 @@ class WebViewAudioManager( * This class is used to handle the audio device selection in the WebView. * It listens for the audio device selection event and calls the callback with the selected device ID. */ -private class WebViewAudioOutputCallback( - private val callback: (String) -> Unit, +private class AndroidWebViewAudioBridge( + private val onAudioDeviceSelected: (String) -> Unit, + private val onAudioTrackReady: () -> Unit, ) { @JavascriptInterface fun setOutputDevice(id: String) { Timber.d("Audio device selected in webview, id: $id") - callback(id) + onAudioDeviceSelected(id) + } + + @JavascriptInterface + fun onTrackReady() { + // This method can be used to notify the WebView that the audio track is ready + // It can be used to start playing audio or to update the UI + Timber.d("Audio track is ready") + + onAudioTrackReady() } } From 4446e337300b616701b153662ae391e968ba668b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Thu, 5 Jun 2025 16:29:18 +0200 Subject: [PATCH 08/11] Replace the `onAudioTrackReady` callback with `onAudioPlaybackStarted` to keep up to date with Element Call --- .../features/call/impl/utils/WebViewAudioManager.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt index 18344b37e1d..294d1190a32 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt @@ -213,7 +213,7 @@ class WebViewAudioManager( previousSelectedDevice = listAudioDevices().find { it.id.toString() == selectedDeviceId } audioManager.selectAudioDevice(selectedDeviceId) }, - onAudioTrackReady = { + onAudioPlaybackStarted = { MainScope().launch { // Calling this ahead of time makes the default audio device to not use the right audio stream setAvailableAudioDevices() @@ -239,8 +239,8 @@ class WebViewAudioManager( * It should be called with some delay after [registerWebViewDeviceSelectedCallback]. */ private fun setWebViewAndroidNativeBridge() { - Timber.d("Adding callback in controls.onAudioTrackReady") - webView.evaluateJavascript("controls.onAudioTrackReady = () => { console.log('SET TRACK READY SET'); androidNativeBridge.onTrackReady(); };", null) + Timber.d("Adding callback in controls.onAudioPlaybackStarted") + webView.evaluateJavascript("controls.onAudioPlaybackStarted = () => { console.log('SET TRACK READY SET'); androidNativeBridge.onTrackReady(); };", null) Timber.d("Adding callback in controls.onOutputDeviceSelect") webView.evaluateJavascript("controls.onOutputDeviceSelect = (id) => { androidNativeBridge.setOutputDevice(id); };", null) } @@ -374,7 +374,7 @@ class WebViewAudioManager( */ private class AndroidWebViewAudioBridge( private val onAudioDeviceSelected: (String) -> Unit, - private val onAudioTrackReady: () -> Unit, + private val onAudioPlaybackStarted: () -> Unit, ) { @JavascriptInterface fun setOutputDevice(id: String) { @@ -388,7 +388,7 @@ private class AndroidWebViewAudioBridge( // It can be used to start playing audio or to update the UI Timber.d("Audio track is ready") - onAudioTrackReady() + onAudioPlaybackStarted() } } From bf619b62ccd6f5d229509434ce5cef6080fa7ac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Fri, 6 Jun 2025 08:38:41 +0200 Subject: [PATCH 09/11] Fix rebase issues --- .../element/android/libraries/androidutils/compat/AudioManager.kt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/compat/AudioManager.kt diff --git a/libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/compat/AudioManager.kt b/libraries/androidutils/src/main/kotlin/io/element/android/libraries/androidutils/compat/AudioManager.kt deleted file mode 100644 index e69de29bb2d..00000000000 From d369cafa4b9f8ea1c6a67ebe375fda103e259d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Fri, 6 Jun 2025 09:29:01 +0200 Subject: [PATCH 10/11] Provide coroutine scope to `WebViewAudioManager` --- .../android/features/call/impl/ui/CallScreenView.kt | 4 +++- .../features/call/impl/utils/WebViewAudioManager.kt | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt index c0997c617eb..f1aa192d289 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/ui/CallScreenView.kt @@ -24,6 +24,7 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier @@ -103,6 +104,7 @@ internal fun CallScreenView( ) } else { var webViewAudioManager by remember { mutableStateOf(null) } + val coroutineScope = rememberCoroutineScope() CallWebView( modifier = Modifier .padding(padding) @@ -128,7 +130,7 @@ internal fun CallScreenView( }, onError = { state.eventSink(CallScreenEvents.OnWebViewError(it)) }, ) - webViewAudioManager = WebViewAudioManager(webView) + webViewAudioManager = WebViewAudioManager(webView, coroutineScope) state.eventSink(CallScreenEvents.SetupMessageChannels(interceptor)) val pipController = WebViewPipController(webView) pipState.eventSink(PictureInPictureEvents.SetPipController(pipController)) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt index 294d1190a32..e74e2bbf5e1 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt @@ -16,7 +16,8 @@ import android.os.PowerManager import android.webkit.JavascriptInterface import android.webkit.WebView import androidx.core.content.getSystemService -import kotlinx.coroutines.MainScope +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.serialization.Serializable import kotlinx.serialization.Transient @@ -35,6 +36,7 @@ import java.util.concurrent.atomic.AtomicBoolean */ class WebViewAudioManager( private val webView: WebView, + private val coroutineScope: CoroutineScope, ) { // The list of device types that are considered as communication devices, sorted by likelihood of it being used for communication. private val wantedDeviceTypes = listOf( @@ -214,7 +216,7 @@ class WebViewAudioManager( audioManager.selectAudioDevice(selectedDeviceId) }, onAudioPlaybackStarted = { - MainScope().launch { + coroutineScope.launch(Dispatchers.Main) { // Calling this ahead of time makes the default audio device to not use the right audio stream setAvailableAudioDevices() @@ -307,7 +309,9 @@ class WebViewAudioManager( * @param deviceId The id of the selected audio device. */ private fun updateSelectedAudioDeviceInWebView(deviceId: String) { - MainScope().launch { webView.evaluateJavascript("controls.setOutputDevice('$deviceId');", null) } + coroutineScope.launch(Dispatchers.Main) { + webView.evaluateJavascript("controls.setOutputDevice('$deviceId');", null) + } } /** From d996cf23563d09a822123892759b69bd782bcf20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Mart=C3=ADn?= Date: Fri, 6 Jun 2025 15:25:53 +0200 Subject: [PATCH 11/11] Remove `console.log` previously added for debugging --- .../android/features/call/impl/utils/WebViewAudioManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt index e74e2bbf5e1..d29843a70e0 100644 --- a/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt +++ b/features/call/impl/src/main/kotlin/io/element/android/features/call/impl/utils/WebViewAudioManager.kt @@ -242,7 +242,7 @@ class WebViewAudioManager( */ private fun setWebViewAndroidNativeBridge() { Timber.d("Adding callback in controls.onAudioPlaybackStarted") - webView.evaluateJavascript("controls.onAudioPlaybackStarted = () => { console.log('SET TRACK READY SET'); androidNativeBridge.onTrackReady(); };", null) + webView.evaluateJavascript("controls.onAudioPlaybackStarted = () => { androidNativeBridge.onTrackReady(); };", null) Timber.d("Adding callback in controls.onOutputDeviceSelect") webView.evaluateJavascript("controls.onOutputDeviceSelect = (id) => { androidNativeBridge.setOutputDevice(id); };", null) }