Skip to content

Commit e691ae0

Browse files
jsaundjsaund
authored
Merge pull request #601 from jsaund/camerax-extensions-latency-estimate
Add Capture Latency Estiamte Indicator for Extension
2 parents d0c49c2 + 227b01a commit e691ae0

File tree

8 files changed

+238
-8
lines changed

8 files changed

+238
-8
lines changed

‎CameraXExtensions/app/src/main/java/com/example/android/cameraxextensions/MainActivity.kt

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import android.util.Log
2424
import androidx.activity.OnBackPressedCallback
2525
import androidx.activity.result.contract.ActivityResultContracts
2626
import androidx.appcompat.app.AppCompatActivity
27+
import androidx.camera.core.ImageCaptureLatencyEstimate
2728
import androidx.camera.extensions.ExtensionMode
2829
import androidx.core.app.ActivityCompat
2930
import androidx.lifecycle.Lifecycle
@@ -53,6 +54,11 @@ import kotlinx.coroutines.launch
5354
* button will capture a photo and display the photo.
5455
*/
5556
class MainActivity : AppCompatActivity() {
57+
private companion object {
58+
const val TAG = "MainActivity"
59+
const val CAPTURE_LATENCY_INDICATOR_THRESHOLD_MS = 500
60+
}
61+
5662
private val extensionName = mapOf(
5763
ExtensionMode.AUTO to R.string.camera_mode_auto,
5864
ExtensionMode.NIGHT to R.string.camera_mode_night,
@@ -308,6 +314,7 @@ class MainActivity : AppCompatActivity() {
308314
cameraExtensionsViewModel.initializeCamera()
309315
}
310316
CameraState.READY -> {
317+
Log.d(TAG, "Camera is ready")
311318
cameraExtensionsScreen.previewView.doOnLaidOut {
312319
cameraExtensionsViewModel.startPreview(
313320
this@MainActivity as LifecycleOwner,
@@ -330,6 +337,22 @@ class MainActivity : AppCompatActivity() {
330337
}
331338
)
332339
}
340+
CameraState.PREVIEW_ACTIVE -> {
341+
Log.d(TAG, "Camera preview is active")
342+
captureScreenViewState.emit(captureScreenViewState.value.updateCameraScreen { s ->
343+
val latencyEstimate = cameraUiState.realtimeCaptureLatencyEstimate
344+
if (latencyEstimate == ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY) {
345+
Log.d(TAG, "Camera preview is active: hide latency estimate indicator")
346+
s.hideLatencyEstimateIndicator()
347+
} else if (latencyEstimate.captureLatencyMillis <= CAPTURE_LATENCY_INDICATOR_THRESHOLD_MS) {
348+
Log.d(TAG, "Camera preview is active: hide latency estimate indicator (under ${CAPTURE_LATENCY_INDICATOR_THRESHOLD_MS}ms)")
349+
s.hideLatencyEstimateIndicator()
350+
} else {
351+
Log.d(TAG, "Camera preview is active: show latency estimate indicator (${latencyEstimate.captureLatencyMillis}ms")
352+
s.showLatencyEstimateIndicator(latencyEstimate.captureLatencyMillis)
353+
}
354+
})
355+
}
333356
CameraState.PREVIEW_STOPPED -> Unit
334357
}
335358
}

‎CameraXExtensions/app/src/main/java/com/example/android/cameraxextensions/model/CameraUiState.kt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import android.graphics.Bitmap
2020
import androidx.camera.core.CameraSelector.LENS_FACING_BACK
2121
import androidx.camera.core.CameraSelector.LensFacing
2222
import androidx.camera.core.ImageCapture
23+
import androidx.camera.core.ImageCaptureLatencyEstimate
2324
import androidx.camera.extensions.ExtensionMode
2425

2526
/**
@@ -33,6 +34,8 @@ data class CameraUiState(
3334
val availableCameraLens: List<Int> = listOf(LENS_FACING_BACK),
3435
@LensFacing val cameraLens: Int = LENS_FACING_BACK,
3536
@ExtensionMode.Mode val extensionMode: Int = ExtensionMode.NONE,
37+
val realtimeCaptureLatencyEstimate: ImageCaptureLatencyEstimate =
38+
ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY
3639
)
3740

3841
/**
@@ -49,6 +52,12 @@ enum class CameraState {
4952
*/
5053
READY,
5154

55+
/**
56+
* Camera is open and preview stream is currently running.
57+
* Updates during this period are from camera state operations.
58+
*/
59+
PREVIEW_ACTIVE,
60+
5261
/**
5362
* Camera is initialized but the preview has been stopped.
5463
*/

‎CameraXExtensions/app/src/main/java/com/example/android/cameraxextensions/ui/CameraExtensionsScreen.kt

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,9 @@ import com.google.android.material.progressindicator.CircularProgressIndicator
5353
import kotlinx.coroutines.flow.Flow
5454
import kotlinx.coroutines.flow.MutableSharedFlow
5555
import kotlinx.coroutines.launch
56+
import java.util.concurrent.TimeUnit
5657
import kotlin.math.max
58+
import kotlin.math.roundToInt
5759

5860
/**
5961
* Displays the camera preview and captured photo.
@@ -90,6 +92,8 @@ class CameraExtensionsScreen(private val root: View) {
9092
root.findViewById(R.id.processProgressContainer)
9193
private val processProgressIndicator: CircularProgressIndicator =
9294
root.findViewById(R.id.processProgressIndicator)
95+
private val latencyEstimateIndicator: TextView =
96+
root.findViewById(R.id.latencyEstimateIndicator)
9397

9498
val previewView: PreviewView = root.findViewById(R.id.previewView)
9599

@@ -264,6 +268,77 @@ class CameraExtensionsScreen(private val root: View) {
264268
processProgressIndicator.progress = 0
265269
}
266270

271+
private fun showLatencyEstimate(latencyEstimateMillis: Long) {
272+
val estimateSeconds = (latencyEstimateMillis.toFloat() / 1000).roundToInt()
273+
274+
if (!latencyEstimateIndicator.isVisible) {
275+
val alphaAnimation =
276+
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.ALPHA, 1f).apply {
277+
spring.stiffness = SpringForce.STIFFNESS_LOW
278+
spring.dampingRatio = SpringForce.DAMPING_RATIO_NO_BOUNCY
279+
}
280+
281+
val scaleAnimationX =
282+
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_X, 1f).apply {
283+
spring.stiffness = SpringForce.STIFFNESS_LOW
284+
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
285+
}
286+
287+
val scaleAnimationY =
288+
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_Y, 1f).apply {
289+
spring.stiffness = SpringForce.STIFFNESS_LOW
290+
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
291+
}
292+
293+
latencyEstimateIndicator.apply {
294+
isVisible = true
295+
alpha = 0f
296+
scaleX = 0.2f
297+
scaleY = 0.2f
298+
}
299+
300+
alphaAnimation.start()
301+
scaleAnimationX.start()
302+
scaleAnimationY.start()
303+
}
304+
305+
latencyEstimateIndicator.text =
306+
context.getString(R.string.latency_estimate, estimateSeconds)
307+
}
308+
309+
private fun hideLatencyEstimate() {
310+
if (latencyEstimateIndicator.isVisible) {
311+
val alphaAnimation =
312+
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.ALPHA, 0f).apply {
313+
spring.stiffness = SpringForce.STIFFNESS_LOW
314+
spring.dampingRatio = SpringForce.DAMPING_RATIO_NO_BOUNCY
315+
316+
addEndListener { _, canceled, _, _ ->
317+
if (!canceled) {
318+
latencyEstimateIndicator.isVisible = false
319+
latencyEstimateIndicator.text = ""
320+
}
321+
}
322+
}
323+
324+
val scaleAnimationX =
325+
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_X, 0f).apply {
326+
spring.stiffness = SpringForce.STIFFNESS_LOW
327+
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
328+
}
329+
330+
val scaleAnimationY =
331+
SpringAnimation(latencyEstimateIndicator, DynamicAnimation.SCALE_Y, 0f).apply {
332+
spring.stiffness = SpringForce.STIFFNESS_LOW
333+
spring.dampingRatio = SpringForce.DAMPING_RATIO_LOW_BOUNCY
334+
}
335+
336+
alphaAnimation.start()
337+
scaleAnimationX.start()
338+
scaleAnimationY.start()
339+
}
340+
}
341+
267342
private fun showPhoto(uri: Uri?) {
268343
if (uri == null) return
269344
photoPreview.isVisible = true
@@ -300,6 +375,12 @@ class CameraExtensionsScreen(private val root: View) {
300375
} else {
301376
hideProcessProgressIndicator()
302377
}
378+
379+
if (state.latencyEstimateIndicatorViewState.isVisible) {
380+
showLatencyEstimate(state.latencyEstimateIndicatorViewState.latencyEstimateMillis)
381+
} else {
382+
hideLatencyEstimate()
383+
}
303384
}
304385

305386
private fun onItemClick(view: View) {

‎CameraXExtensions/app/src/main/java/com/example/android/cameraxextensions/viewmodel/CameraExtensionsViewModel.kt

Lines changed: 81 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@ package com.example.android.cameraxextensions.viewmodel
1818

1919
import android.app.Application
2020
import android.graphics.Bitmap
21+
import android.util.Log
2122
import androidx.camera.core.AspectRatio
2223
import androidx.camera.core.Camera
2324
import androidx.camera.core.CameraSelector
2425
import androidx.camera.core.CameraSelector.LensFacing
2526
import androidx.camera.core.FocusMeteringAction
2627
import androidx.camera.core.ImageCapture
2728
import androidx.camera.core.ImageCaptureException
29+
import androidx.camera.core.ImageCaptureLatencyEstimate
2830
import androidx.camera.core.MeteringPoint
2931
import androidx.camera.core.Preview
3032
import androidx.camera.core.UseCaseGroup
@@ -41,11 +43,16 @@ import com.example.android.cameraxextensions.model.CameraUiState
4143
import com.example.android.cameraxextensions.model.CaptureState
4244
import com.example.android.cameraxextensions.repository.ImageCaptureRepository
4345
import kotlinx.coroutines.Dispatchers
46+
import kotlinx.coroutines.Job
4447
import kotlinx.coroutines.asExecutor
48+
import kotlinx.coroutines.delay
4549
import kotlinx.coroutines.flow.Flow
4650
import kotlinx.coroutines.flow.MutableStateFlow
4751
import kotlinx.coroutines.guava.await
52+
import kotlinx.coroutines.isActive
4853
import kotlinx.coroutines.launch
54+
import androidx.lifecycle.asFlow
55+
import kotlinx.coroutines.CoroutineScope
4956

5057
/**
5158
* View model for camera extensions. This manages all the operations on the camera.
@@ -61,6 +68,11 @@ class CameraExtensionsViewModel(
6168
private val application: Application,
6269
private val imageCaptureRepository: ImageCaptureRepository
6370
) : ViewModel() {
71+
private companion object {
72+
const val TAG = "CameraExtensionsViewModel"
73+
const val REALTIME_LATENCY_UPDATE_INTERVAL_MILLIS = 1000L
74+
}
75+
6476
private lateinit var cameraProvider: ProcessCameraProvider
6577
private lateinit var extensionsManager: ExtensionsManager
6678

@@ -69,6 +81,7 @@ class CameraExtensionsViewModel(
6981
private var imageCapture = ImageCapture.Builder()
7082
.setTargetAspectRatio(AspectRatio.RATIO_16_9)
7183
.build()
84+
private var realtimeLatencyEstimateJob: Job? = null
7285

7386
private val preview = Preview.Builder()
7487
.setTargetAspectRatio(AspectRatio.RATIO_16_9)
@@ -127,6 +140,7 @@ class CameraExtensionsViewModel(
127140
availableExtensions = listOf(ExtensionMode.NONE) + availableExtensions,
128141
availableCameraLens = availableCameraLens,
129142
extensionMode = if (availableExtensions.isEmpty()) ExtensionMode.NONE else currentCameraUiState.extensionMode,
143+
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY,
130144
)
131145
_cameraUiState.emit(newCameraUiState)
132146
}
@@ -141,6 +155,8 @@ class CameraExtensionsViewModel(
141155
lifecycleOwner: LifecycleOwner,
142156
previewView: PreviewView
143157
) {
158+
realtimeLatencyEstimateJob?.cancel()
159+
144160
val currentCameraUiState = _cameraUiState.value
145161
val cameraSelector = if (currentCameraUiState.extensionMode == ExtensionMode.NONE) {
146162
cameraLensToSelector(currentCameraUiState.cameraLens)
@@ -175,30 +191,75 @@ class CameraExtensionsViewModel(
175191
useCaseGroup
176192
)
177193

178-
preview.setSurfaceProvider(previewView.surfaceProvider)
194+
preview.surfaceProvider = previewView.surfaceProvider
179195

180196
viewModelScope.launch {
181197
_cameraUiState.emit(_cameraUiState.value.copy(cameraState = CameraState.READY))
182198
_captureUiState.emit(CaptureState.CaptureReady)
199+
previewView.previewStreamState.asFlow().collect { previewStreamState ->
200+
when (previewStreamState) {
201+
PreviewView.StreamState.IDLE -> {
202+
realtimeLatencyEstimateJob?.cancel()
203+
realtimeLatencyEstimateJob = null
204+
}
205+
PreviewView.StreamState.STREAMING -> {
206+
if (realtimeLatencyEstimateJob == null) {
207+
realtimeLatencyEstimateJob = launch {
208+
observeRealtimeLatencyEstimate()
209+
}
210+
}
211+
}
212+
}
213+
}
214+
}
215+
}
216+
217+
private suspend fun CoroutineScope.observeRealtimeLatencyEstimate() {
218+
Log.d(TAG, "Starting realtime latency estimate job")
219+
220+
val currentCameraUiState = _cameraUiState.value
221+
val isSupported =
222+
currentCameraUiState.extensionMode != ExtensionMode.NONE
223+
&& imageCapture.realtimeCaptureLatencyEstimate != ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY
224+
225+
if (!isSupported) {
226+
Log.d(TAG, "Starting realtime latency estimate job: no extension mode or not supported")
227+
_cameraUiState.emit(
228+
_cameraUiState.value.copy(
229+
cameraState = CameraState.PREVIEW_ACTIVE,
230+
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY
231+
)
232+
)
233+
return
234+
}
235+
236+
while (isActive) {
237+
updateRealtimeCaptureLatencyEstimate()
238+
delay(REALTIME_LATENCY_UPDATE_INTERVAL_MILLIS)
183239
}
184240
}
185241

186242
/**
187243
* Stops the preview stream. This should be invoked when the captured image is displayed.
188244
*/
189245
fun stopPreview() {
190-
preview.setSurfaceProvider(null)
246+
realtimeLatencyEstimateJob?.cancel()
247+
preview.surfaceProvider = null
191248
viewModelScope.launch {
192-
_cameraUiState.emit(_cameraUiState.value.copy(cameraState = CameraState.PREVIEW_STOPPED))
249+
_cameraUiState.emit(_cameraUiState.value.copy(
250+
cameraState = CameraState.PREVIEW_STOPPED,
251+
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY
252+
))
193253
}
194254
}
195255

196256
/**
197257
* Toggle the camera lens face. This has no effect if there is only one available camera lens.
198258
*/
199259
fun switchCamera() {
260+
realtimeLatencyEstimateJob?.cancel()
200261
val currentCameraUiState = _cameraUiState.value
201-
if (currentCameraUiState.cameraState == CameraState.READY) {
262+
if (currentCameraUiState.cameraState == CameraState.READY || currentCameraUiState.cameraState == CameraState.PREVIEW_ACTIVE) {
202263
// To switch the camera lens, there has to be at least 2 camera lenses
203264
if (currentCameraUiState.availableCameraLens.size == 1) return
204265

@@ -230,6 +291,7 @@ class CameraExtensionsViewModel(
230291
* exception containing more details on the reason for failure.
231292
*/
232293
fun capturePhoto() {
294+
realtimeLatencyEstimateJob?.cancel()
233295
viewModelScope.launch {
234296
_captureUiState.emit(CaptureState.CaptureStarted)
235297
}
@@ -306,6 +368,7 @@ class CameraExtensionsViewModel(
306368
_cameraUiState.value.copy(
307369
cameraState = CameraState.NOT_READY,
308370
extensionMode = extensionMode,
371+
realtimeCaptureLatencyEstimate = ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY,
309372
)
310373
)
311374
_captureUiState.emit(CaptureState.CaptureNotReady)
@@ -331,4 +394,18 @@ class CameraExtensionsViewModel(
331394
CameraSelector.LENS_FACING_BACK -> CameraSelector.DEFAULT_BACK_CAMERA
332395
else -> throw IllegalArgumentException("Invalid lens facing type: $lensFacing")
333396
}
397+
398+
private suspend fun updateRealtimeCaptureLatencyEstimate() {
399+
val estimate = imageCapture.realtimeCaptureLatencyEstimate
400+
Log.d(TAG, "Realtime capture latency estimate: $estimate")
401+
if (estimate == ImageCaptureLatencyEstimate.UNDEFINED_IMAGE_CAPTURE_LATENCY) {
402+
return
403+
}
404+
_cameraUiState.emit(
405+
_cameraUiState.value.copy(
406+
cameraState = CameraState.PREVIEW_ACTIVE,
407+
realtimeCaptureLatencyEstimate = estimate
408+
)
409+
)
410+
}
334411
}

0 commit comments

Comments
 (0)