Retry setting video encoder when starting video recording
Recorder may fail to set video encoder in the initial state and enter an error state. One case is when the codec resource is occupied by a previous Recorder that is in a releasing state, or even occupied by another process. This change will try to reconfigure the video encoder when the recording request is made.
Relnote: "Fixed a crash when the Recorder encountered an InvalidConfigException. However, this fix only prevents app from crashing, but doesn't resolve the cause of the InvalidConfigException. If the Recorder still cannot be configured, applications will receive error callback when it starts recording."
Bug: 200359121
Bug: 213617227
Test: ./gradlew camera:camera-video:connectedAndroidTest
Change-Id: I89c295716c6228c47f8bdadefef8336c9c058bd3
diff --git a/camera/camera-video/src/androidTest/java/androidx/camera/video/RecorderTest.kt b/camera/camera-video/src/androidTest/java/androidx/camera/video/RecorderTest.kt
index 29ed1e2..ef032ce 100644
--- a/camera/camera-video/src/androidTest/java/androidx/camera/video/RecorderTest.kt
+++ b/camera/camera-video/src/androidTest/java/androidx/camera/video/RecorderTest.kt
@@ -43,11 +43,14 @@
import androidx.camera.testing.GarbageCollectionUtil
import androidx.camera.testing.LabTestRule
import androidx.camera.testing.SurfaceTextureProvider
+import androidx.camera.testing.asFlow
import androidx.camera.video.VideoRecordEvent.Finalize.ERROR_FILE_SIZE_LIMIT_REACHED
import androidx.camera.video.VideoRecordEvent.Finalize.ERROR_INVALID_OUTPUT_OPTIONS
+import androidx.camera.video.VideoRecordEvent.Finalize.ERROR_RECORDER_ERROR
import androidx.camera.video.VideoRecordEvent.Finalize.ERROR_SOURCE_INACTIVE
import androidx.camera.video.internal.compat.quirk.DeactivateEncoderSurfaceBeforeStopEncoderQuirk
import androidx.camera.video.internal.compat.quirk.DeviceQuirks
+import androidx.camera.video.internal.encoder.InvalidConfigException
import androidx.core.util.Consumer
import androidx.test.core.app.ApplicationProvider
import androidx.test.ext.junit.runners.AndroidJUnit4
@@ -56,7 +59,16 @@
import androidx.test.platform.app.InstrumentationRegistry
import androidx.test.rule.GrantPermissionRule
import androidx.testutils.assertThrows
+import androidx.testutils.fail
import com.google.common.truth.Truth.assertThat
+import java.io.File
+import java.util.concurrent.Executor
+import java.util.concurrent.Semaphore
+import java.util.concurrent.TimeUnit
+import kotlinx.coroutines.flow.dropWhile
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeoutOrNull
import org.junit.After
import org.junit.Assume.assumeFalse
import org.junit.Assume.assumeTrue
@@ -77,10 +89,6 @@
import org.mockito.Mockito.timeout
import org.mockito.Mockito.verify
import org.mockito.Mockito.verifyNoMoreInteractions
-import java.io.File
-import java.util.concurrent.Executor
-import java.util.concurrent.Semaphore
-import java.util.concurrent.TimeUnit
private const val FINALIZE_TIMEOUT = 5000L
@@ -1066,7 +1074,67 @@
}
}
+ @Test
+ fun canRecoverFromErrorState(): Unit = runBlocking {
+ // Create a video encoder factory that will fail on first 2 create encoder requests.
+ // Recorder initialization should fail by 1st encoder creation fail.
+ // 1st recording request should fail by 2nd encoder creation fail.
+ // 2nd recording request should be successful.
+ var createEncoderRequestCount = 0
+ val recorder = Recorder.Builder()
+ .setVideoEncoderFactory { executor, config ->
+ if (createEncoderRequestCount < 2) {
+ createEncoderRequestCount++
+ throw InvalidConfigException("Create video encoder fail on purpose.")
+ } else {
+ Recorder.DEFAULT_ENCODER_FACTORY.createEncoder(executor, config)
+ }
+ }.build().apply { onSourceStateChanged(VideoOutput.SourceState.INACTIVE) }
+
+ invokeSurfaceRequest(recorder)
+ val file = File.createTempFile("CameraX", ".tmp").apply { deleteOnExit() }
+
+ // Wait STREAM_ID_ERROR which indicates Recorder enter the error state.
+ withTimeoutOrNull(3000) {
+ recorder.streamInfo.asFlow().dropWhile { it!!.id != StreamInfo.STREAM_ID_ERROR }.first()
+ } ?: fail("Do not observe STREAM_ID_ERROR from StreamInfo observer.")
+
+ // 1st recording request
+ clearInvocations(videoRecordEventListener)
+ recorder.prepareRecording(context, FileOutputOptions.Builder(file).build())
+ .withAudioEnabled()
+ .start(CameraXExecutors.directExecutor(), videoRecordEventListener).let {
+ val captor = ArgumentCaptor.forClass(VideoRecordEvent::class.java)
+ verify(videoRecordEventListener, timeout(3000)).accept(captor.capture())
+ val finalize = captor.value as VideoRecordEvent.Finalize
+ assertThat(finalize.error).isEqualTo(ERROR_RECORDER_ERROR)
+ }
+
+ // 2nd recording request
+ clearInvocations(videoRecordEventListener)
+ recorder.prepareRecording(context, FileOutputOptions.Builder(file).build())
+ .withAudioEnabled()
+ .start(CameraXExecutors.directExecutor(), videoRecordEventListener).let {
+ val inOrder = inOrder(videoRecordEventListener)
+ inOrder.verify(videoRecordEventListener, timeout(3000L))
+ .accept(any(VideoRecordEvent.Start::class.java))
+ inOrder.verify(videoRecordEventListener, timeout(15000L).atLeast(5))
+ .accept(any(VideoRecordEvent.Status::class.java))
+
+ it.stopSafely()
+
+ inOrder.verify(videoRecordEventListener, timeout(FINALIZE_TIMEOUT))
+ .accept(any(VideoRecordEvent.Finalize::class.java))
+ }
+
+ file.delete()
+ }
+
private fun invokeSurfaceRequest() {
+ invokeSurfaceRequest(recorder)
+ }
+
+ private fun invokeSurfaceRequest(recorder: Recorder) {
instrumentation.runOnMainSync {
preview.setSurfaceProvider { request: SurfaceRequest ->
recorder.onSurfaceRequested(request)
diff --git a/camera/camera-video/src/main/java/androidx/camera/video/Recorder.java b/camera/camera-video/src/main/java/androidx/camera/video/Recorder.java
index 297b2e2..c799b8f 100644
--- a/camera/camera-video/src/main/java/androidx/camera/video/Recorder.java
+++ b/camera/camera-video/src/main/java/androidx/camera/video/Recorder.java
@@ -46,6 +46,7 @@
import androidx.annotation.RequiresApi;
import androidx.annotation.RequiresPermission;
import androidx.annotation.RestrictTo;
+import androidx.annotation.VisibleForTesting;
import androidx.camera.core.AspectRatio;
import androidx.camera.core.Logger;
import androidx.camera.core.SurfaceRequest;
@@ -78,6 +79,7 @@
import androidx.camera.video.internal.encoder.EncodedData;
import androidx.camera.video.internal.encoder.Encoder;
import androidx.camera.video.internal.encoder.EncoderCallback;
+import androidx.camera.video.internal.encoder.EncoderFactory;
import androidx.camera.video.internal.encoder.EncoderImpl;
import androidx.camera.video.internal.encoder.InvalidConfigException;
import androidx.camera.video.internal.encoder.OutputConfig;
@@ -235,7 +237,8 @@
State.INITIALIZING, // Waiting for camera before starting recording.
State.IDLING, // Waiting for sequential executor to start pending recording.
State.RESETTING, // Waiting for camera/encoders to reset before starting.
- State.STOPPING // Waiting for previous recording to finalize before starting.
+ State.STOPPING, // Waiting for previous recording to finalize before starting.
+ State.ERROR // Waiting for re-initialization before starting.
));
/**
@@ -272,6 +275,8 @@
private static final int PENDING = 1;
private static final int NOT_PENDING = 0;
private static final long SOURCE_NON_STREAMING_TIMEOUT = 1000L;
+ @VisibleForTesting
+ static final EncoderFactory DEFAULT_ENCODER_FACTORY = EncoderImpl::new;
private final MutableStateObservable<StreamInfo> mStreamInfo;
// Used only by getExecutor()
@@ -281,6 +286,8 @@
private final Executor mExecutor;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
final Executor mSequentialExecutor;
+ private final EncoderFactory mVideoEncoderFactory;
+ private final EncoderFactory mAudioEncoderFactory;
private final Object mLock = new Object();
////////////////////////////////////////////////////////////////////////////////////////////////
@@ -292,6 +299,9 @@
// should be null.
@GuardedBy("mLock")
private State mNonPendingState = null;
+ @SuppressWarnings("WeakerAccess") /* synthetic accessor */
+ @GuardedBy("mLock")
+ int mStreamId = StreamInfo.STREAM_ID_ANY;
@GuardedBy("mLock")
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
RecordingRecord mActiveRecordingRecord = null;
@@ -303,8 +313,6 @@
@GuardedBy("mLock")
private SourceState mSourceState = SourceState.INACTIVE;
@GuardedBy("mLock")
- private Throwable mErrorCause;
- @GuardedBy("mLock")
private long mLastGeneratedRecordingId = 0L;
@GuardedBy("mLock")
private CallbackToFutureAdapter.Completer<Void> mSourceNonStreamingCompleter = null;
@@ -326,6 +334,8 @@
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
Integer mVideoTrackIndex = null;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
+ SurfaceRequest mSurfaceRequest;
+ @SuppressWarnings("WeakerAccess") /* synthetic accessor */
Surface mSurface = null;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
MediaMuxer mMediaMuxer = null;
@@ -334,11 +344,11 @@
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
AudioSource mAudioSource = null;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
- EncoderImpl mVideoEncoder = null;
+ Encoder mVideoEncoder = null;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
OutputConfig mVideoOutputConfig = null;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
- EncoderImpl mAudioEncoder = null;
+ Encoder mAudioEncoder = null;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
OutputConfig mAudioOutputConfig = null;
@SuppressWarnings("WeakerAccess") /* synthetic accessor */
@@ -368,21 +378,25 @@
boolean mIsAudioSourceSilenced = false;
//--------------------------------------------------------------------------------------------//
- Recorder(@Nullable Executor executor, @NonNull MediaSpec mediaSpec) {
+ Recorder(@Nullable Executor executor, @NonNull MediaSpec mediaSpec,
+ @NonNull EncoderFactory videoEncoderFactory,
+ @NonNull EncoderFactory audioEncoderFactory) {
mUserProvidedExecutor = executor;
mExecutor = executor != null ? executor : CameraXExecutors.ioExecutor();
mSequentialExecutor = CameraXExecutors.newSequentialExecutor(mExecutor);
mMediaSpec = MutableStateObservable.withInitialState(composeRecorderMediaSpec(mediaSpec));
mStreamInfo = MutableStateObservable.withInitialState(
- StreamInfo.of(generateStreamId(), internalStateToStreamState(mState)));
+ StreamInfo.of(mStreamId, internalStateToStreamState(mState)));
+ mVideoEncoderFactory = videoEncoderFactory;
+ mAudioEncoderFactory = audioEncoderFactory;
}
@Override
public void onSurfaceRequested(@NonNull SurfaceRequest request) {
synchronized (mLock) {
Logger.d(TAG, "Surface is requested in state: " + mState + ", Current surface: "
- + generateStreamId());
+ + mStreamId);
switch (mState) {
case STOPPING:
// Fall-through
@@ -393,7 +407,8 @@
case PENDING_PAUSED:
// Fall-through
case INITIALIZING:
- mSequentialExecutor.execute(() -> initializeInternal(request));
+ mSequentialExecutor.execute(
+ () -> initializeInternal(mSurfaceRequest = request));
break;
case IDLING:
// Fall-through
@@ -403,8 +418,16 @@
throw new IllegalStateException("Surface was requested when the Recorder had "
+ "been initialized with state " + mState);
case ERROR:
- throw new IllegalStateException("Surface was requested when the Recorder had "
- + "encountered error " + mErrorCause);
+ Logger.w(TAG, "Surface was requested when the Recorder had encountered error.");
+ setState(State.INITIALIZING);
+ mSequentialExecutor.execute(() -> {
+ if (mSurfaceRequest != null) {
+ // If the surface request is already complete, this is a no-op.
+ mSurfaceRequest.willNotProvideSurface();
+ }
+ initializeInternal(mSurfaceRequest = request);
+ });
+ break;
}
}
}
@@ -693,6 +716,8 @@
// Fall-through
case INITIALIZING:
// Fall-through
+ case ERROR:
+ // Fall-through
case IDLING:
if (mState == State.IDLING) {
Preconditions.checkState(
@@ -710,6 +735,17 @@
if (mState == State.IDLING) {
setState(State.PENDING_RECORDING);
mSequentialExecutor.execute(this::tryServicePendingRecording);
+ } else if (mState == State.ERROR) {
+ setState(State.PENDING_RECORDING);
+ // Retry initialization.
+ mSequentialExecutor.execute(() -> {
+ if (mSurfaceRequest == null) {
+ throw new AssertionError(
+ "surface request is required to retry "
+ + "initialization.");
+ }
+ initializeInternal(mSurfaceRequest);
+ });
} else {
setState(State.PENDING_RECORDING);
// The recording will automatically start once the initialization
@@ -720,10 +756,6 @@
errorCause = e;
}
break;
- case ERROR:
- error = ERROR_RECORDER_ERROR;
- errorCause = mErrorCause;
- break;
}
}
}
@@ -972,7 +1004,7 @@
}
@ExecutedBy("mSequentialExecutor")
- private void initializeInternal(SurfaceRequest surfaceRequest) {
+ private void initializeInternal(@NonNull SurfaceRequest surfaceRequest) {
if (mSurface != null) {
// There's a valid surface. Provide it directly.
surfaceRequest.provideSurface(mSurface, mSequentialExecutor, this::onSurfaceClosed);
@@ -1028,8 +1060,7 @@
break;
case ERROR:
Logger.e(TAG,
- "onInitialized() was invoked when the Recorder had encountered error "
- + mErrorCause);
+ "onInitialized() was invoked when the Recorder had encountered error");
break;
case PENDING_PAUSED:
startRecordingPaused = true;
@@ -1239,7 +1270,7 @@
AudioEncoderConfig audioEncoderConfig = resolveAudioEncoderConfig(audioMimeInfo,
audioSourceSettings, mediaSpec.getAudioSpec());
try {
- mAudioEncoder = new EncoderImpl(mExecutor, audioEncoderConfig);
+ mAudioEncoder = mAudioEncoderFactory.createEncoder(mExecutor, audioEncoderConfig);
} catch (InvalidConfigException e) {
throw new ResourceCreationException(e);
}
@@ -1292,9 +1323,8 @@
mediaSpec.getVideoSpec(), surfaceRequest.getResolution());
try {
- mVideoEncoder = new EncoderImpl(mExecutor, config);
+ mVideoEncoder = mVideoEncoderFactory.createEncoder(mExecutor, config);
} catch (InvalidConfigException e) {
- surfaceRequest.willNotProvideSurface();
Logger.e(TAG, "Unable to initialize video encoder.", e);
onEncoderSetupError(new ResourceCreationException(e));
return;
@@ -1307,10 +1337,9 @@
((Encoder.SurfaceInput) encoderInput).setOnSurfaceUpdateListener(
mSequentialExecutor,
surface -> {
- Logger.d(TAG,
- "Encoder surface updated: " + surface.hashCode() + ", Current surface: "
- + generateStreamId());
synchronized (mLock) {
+ Logger.d(TAG, "Encoder surface updated: " + surface.hashCode()
+ + ", Current surface: " + mStreamId);
switch (mState) {
case PENDING_RECORDING:
// Fall-through
@@ -1345,19 +1374,15 @@
@NonNull SurfaceRequest surfaceRequest) {
if (mSurface != surface) {
Surface currentSurface = mSurface;
- mSurface = surface;
+ setSurface(surface);
if (currentSurface == null) {
// Provide the surface to the first surface request.
surfaceRequest.provideSurface(surface, mSequentialExecutor, this::onSurfaceClosed);
onInitialized();
} else {
- // Encoder updates the surface while there's already an active surface. Update
- // the StreamInfo with the new stream ID, which will trigger VideoCapture to send
- // a new surface request.
- synchronized (mLock) {
- mStreamInfo.setState(
- StreamInfo.of(generateStreamId(), internalStateToStreamState(mState)));
- }
+ // Encoder updates the surface while there's already an active surface.
+ // setSurface() will update the StreamInfo with the new stream ID, which will
+ // trigger VideoCapture to send a new surface request.
}
} else {
Logger.d(TAG, "Video encoder provides the same surface.");
@@ -1366,8 +1391,7 @@
@ExecutedBy("mSequentialExecutor")
private void onSurfaceClosed(@NonNull SurfaceRequest.Result result) {
- Logger.d(TAG, "Surface closed: " + result.getSurface().hashCode() + ", Current surface: "
- + generateStreamId());
+ Logger.d(TAG, "Surface closed: " + result.getSurface().hashCode());
Surface resultSurface = result.getSurface();
// The latest surface will be released by the encoder when encoder is released.
if (mSurface != resultSurface) {
@@ -1375,7 +1399,7 @@
} else {
// Reset the Recorder when the latest surface is terminated.
reset();
- mSurface = null;
+ setSurface(null);
}
}
@@ -1391,8 +1415,8 @@
mPendingRecordingRecord = null;
// Fall-through
case INITIALIZING:
+ setStreamId(StreamInfo.STREAM_ID_ERROR);
setState(State.ERROR);
- mErrorCause = cause;
break;
case ERROR:
// Already in an error state. Ignore new error.
@@ -1909,7 +1933,9 @@
Futures.addCallback(sourceNonStreamingFuture, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void result) {
- mVideoEncoder.signalSourceStopped();
+ if (mVideoEncoder instanceof EncoderImpl) {
+ ((EncoderImpl) mVideoEncoder).signalSourceStopped();
+ }
}
@Override
@@ -1920,7 +1946,9 @@
// Even in the case of error, we tell the encoder the source has stopped
// because devices with this quirk require that the codec produce a new
// surface.
- mVideoEncoder.signalSourceStopped();
+ if (mVideoEncoder instanceof EncoderImpl) {
+ ((EncoderImpl) mVideoEncoder).signalSourceStopped();
+ }
}
}
}, mSequentialExecutor);
@@ -2344,7 +2372,28 @@
if (streamState == null) {
streamState = internalStateToStreamState(mState);
}
- mStreamInfo.setState(StreamInfo.of(generateStreamId(), streamState));
+ mStreamInfo.setState(StreamInfo.of(mStreamId, streamState));
+ }
+
+ @ExecutedBy("mSequentialExecutor")
+ private void setSurface(@Nullable Surface surface) {
+ if (mSurface == surface) {
+ return;
+ }
+ mSurface = surface;
+ synchronized (mLock) {
+ setStreamId(surface != null ? surface.hashCode() : StreamInfo.STREAM_ID_ANY);
+ }
+ }
+
+ @GuardedBy("mLock")
+ private void setStreamId(int streamId) {
+ if (mStreamId == streamId) {
+ return;
+ }
+ Logger.d(TAG, "Transitioning streamId: " + mStreamId + " --> " + streamId);
+ mStreamId = streamId;
+ mStreamInfo.setState(StreamInfo.of(streamId, internalStateToStreamState(mState)));
}
/**
@@ -2368,7 +2417,7 @@
if (mNonPendingState != state) {
mNonPendingState = state;
mStreamInfo.setState(
- StreamInfo.of(generateStreamId(), internalStateToStreamState(state)));
+ StreamInfo.of(mStreamId, internalStateToStreamState(state)));
}
}
@@ -2416,11 +2465,6 @@
return defaultMuxerFormat;
}
- @ExecutedBy("mSequentialExecutor")
- private Integer generateStreamId() {
- return mSurface == null ? StreamInfo.STREAM_ID_ANY : mSurface.hashCode();
- }
-
@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
@AutoValue
abstract static class RecordingRecord implements AutoCloseable {
@@ -2746,6 +2790,8 @@
private final MediaSpec.Builder mMediaSpecBuilder;
private Executor mExecutor = null;
+ private EncoderFactory mVideoEncoderFactory = DEFAULT_ENCODER_FACTORY;
+ private EncoderFactory mAudioEncoderFactory = DEFAULT_ENCODER_FACTORY;
/**
* Constructor for {@code Recorder.Builder}.
@@ -2823,6 +2869,22 @@
return this;
}
+ /** @hide */
+ @RestrictTo(RestrictTo.Scope.LIBRARY)
+ @NonNull
+ Builder setVideoEncoderFactory(@NonNull EncoderFactory videoEncoderFactory) {
+ mVideoEncoderFactory = videoEncoderFactory;
+ return this;
+ }
+
+ /** @hide */
+ @RestrictTo(RestrictTo.Scope.LIBRARY)
+ @NonNull
+ Builder setAudioEncoderFactory(@NonNull EncoderFactory audioEncoderFactory) {
+ mAudioEncoderFactory = audioEncoderFactory;
+ return this;
+ }
+
/**
* Builds the {@link Recorder} instance.
*
@@ -2832,7 +2894,8 @@
*/
@NonNull
public Recorder build() {
- return new Recorder(mExecutor, mMediaSpecBuilder.build());
+ return new Recorder(mExecutor, mMediaSpecBuilder.build(), mVideoEncoderFactory,
+ mAudioEncoderFactory);
}
}
}
diff --git a/camera/camera-video/src/main/java/androidx/camera/video/StreamInfo.java b/camera/camera-video/src/main/java/androidx/camera/video/StreamInfo.java
index 327bf2a..6c0a522 100644
--- a/camera/camera-video/src/main/java/androidx/camera/video/StreamInfo.java
+++ b/camera/camera-video/src/main/java/androidx/camera/video/StreamInfo.java
@@ -19,7 +19,6 @@
import android.view.Surface;
import androidx.annotation.NonNull;
-import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.annotation.RestrictTo;
import androidx.camera.core.SurfaceRequest;
@@ -28,6 +27,11 @@
import com.google.auto.value.AutoValue;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
/**
* A class that contains the information of an video output stream.
*
@@ -38,11 +42,18 @@
@AutoValue
public abstract class StreamInfo {
- static final Integer STREAM_ID_ANY = 0;
+ /** The stream hasn't been setup. */
+ static final int STREAM_ID_ANY = 0;
+
+ /** The stream setup fails. */
+ static final int STREAM_ID_ERROR = -1;
static final StreamInfo STREAM_INFO_ANY_INACTIVE = StreamInfo.of(STREAM_ID_ANY,
StreamState.INACTIVE);
+ static final Set<Integer> NON_SURFACE_STREAM_ID = Collections.unmodifiableSet(
+ new HashSet<>(Arrays.asList(STREAM_ID_ANY, STREAM_ID_ERROR)));
+
static final Observable<StreamInfo> ALWAYS_ACTIVE_OBSERVABLE =
ConstantObservable.withValue(StreamInfo.of(STREAM_ID_ANY, StreamState.ACTIVE));
/**
@@ -65,7 +76,7 @@
}
@NonNull
- static StreamInfo of(@Nullable Integer id, @NonNull StreamState streamState) {
+ static StreamInfo of(int id, @NonNull StreamState streamState) {
return new AutoValue_StreamInfo(id, streamState);
}
@@ -77,10 +88,10 @@
* {@link SurfaceRequest} has to be issued in order to obtain a new {@link Surface} to
* continue drawing frames to the {@link VideoOutput}.
*
- * <p>The ID will be {@link #STREAM_ID_ANY} if the stream hasn't been setup.
+ * <p>The ID will be {@link #STREAM_ID_ANY} if the stream hasn't been setup and the ID will be
+ * {@link #STREAM_ID_ERROR} if the stream setup fails.
*/
- @NonNull
- public abstract Integer getId();
+ public abstract int getId();
/**
* Gets the stream state which can be used to determine if the video output is ready for
diff --git a/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java b/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java
index 6506e64..df4de24 100644
--- a/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java
+++ b/camera/camera-video/src/main/java/androidx/camera/video/VideoCapture.java
@@ -31,6 +31,7 @@
import static androidx.camera.core.internal.TargetConfig.OPTION_TARGET_NAME;
import static androidx.camera.core.internal.ThreadConfig.OPTION_BACKGROUND_EXECUTOR;
import static androidx.camera.core.internal.UseCaseEventConfig.OPTION_USE_CASE_EVENT_CALLBACK;
+import static androidx.camera.video.StreamInfo.STREAM_ID_ERROR;
import static androidx.camera.video.impl.VideoCaptureConfig.OPTION_VIDEO_OUTPUT;
import android.graphics.Rect;
@@ -222,7 +223,6 @@
*
* @hide
*/
- @SuppressWarnings("unchecked")
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
public void onStateAttached() {
@@ -271,7 +271,10 @@
}
}
+ mStreamInfo = fetchObservableValue(getOutput().getStreamInfo(),
+ StreamInfo.STREAM_INFO_ANY_INACTIVE);
mSessionConfigBuilder = createPipeline(cameraId, config, suggestedResolution);
+ applyStreamInfoToSessionConfigBuilder(mSessionConfigBuilder, mStreamInfo);
updateSessionConfig(mSessionConfigBuilder.build());
// VideoCapture has to be active to apply SessionConfig's template type.
notifyActive();
@@ -309,7 +312,6 @@
*
* @hide
*/
- @SuppressWarnings("unchecked")
@RestrictTo(Scope.LIBRARY_GROUP)
@Override
public void onStateDetached() {
@@ -421,17 +423,8 @@
mDeferrableSurface.setContainerClass(MediaCodec.class);
SessionConfig.Builder sessionConfigBuilder = SessionConfig.Builder.createFrom(config);
- if (fetchObservableValue(getOutput().getStreamInfo(),
- StreamInfo.STREAM_INFO_ANY_INACTIVE).getStreamState() == StreamState.ACTIVE) {
- sessionConfigBuilder.addSurface(mDeferrableSurface);
- getOutput().onSourceStateChanged(VideoOutput.SourceState.ACTIVE_STREAMING);
- } else {
- sessionConfigBuilder.addNonRepeatingSurface(mDeferrableSurface);
- getOutput().onSourceStateChanged(VideoOutput.SourceState.ACTIVE_NON_STREAMING);
- }
sessionConfigBuilder.addErrorListener(
(sessionConfig, error) -> resetPipeline(cameraId, config, resolution));
-
sessionConfigBuilder.addRepeatingCameraCaptureCallback(new CameraCaptureCallback() {
@Override
public void onCaptureCompleted(@NonNull CameraCaptureResult cameraCaptureResult) {
@@ -480,6 +473,7 @@
if (isCurrentCamera(cameraId)) {
// Only reset the pipeline when the bound camera is the same.
mSessionConfigBuilder = createPipeline(cameraId, config, resolution);
+ applyStreamInfoToSessionConfigBuilder(mSessionConfigBuilder, mStreamInfo);
updateSessionConfig(mSessionConfigBuilder.build());
notifyReset();
}
@@ -531,80 +525,35 @@
// VideoCapture is unbound.
return;
}
- if (mStreamInfo.getStreamState() != streamInfo.getStreamState()) {
- mSessionConfigBuilder.clearSurfaces();
- boolean isStreamActive = streamInfo.getStreamState() == StreamState.ACTIVE;
- if (isStreamActive) {
- mSessionConfigBuilder.addSurface(mDeferrableSurface);
- } else {
- mSessionConfigBuilder.addNonRepeatingSurface(mDeferrableSurface);
- }
+ Logger.d(TAG, "Stream info update: old: " + mStreamInfo + " new: " + streamInfo);
- AtomicReference<CallbackToFutureAdapter.Completer<Void>> surfaceUpdateCompleter =
- new AtomicReference<>();
- ListenableFuture<Void> surfaceUpdateFuture =
- CallbackToFutureAdapter.getFuture(completer -> {
- // Use the completer as the tag to identify the update.
- mSessionConfigBuilder.addTag(SURFACE_UPDATE_KEY, completer.hashCode());
- synchronized (mLock) {
- if (mSurfaceUpdateCompleter != null) {
- // A newer update is issued before the previous update is
- // completed. Fail the previous future.
- mSurfaceUpdateCompleter.setException(new RuntimeException(
- "A newer surface update is completed."));
- }
- mSurfaceUpdateCompleter = completer;
- surfaceUpdateCompleter.set(completer);
- }
- return SURFACE_UPDATE_KEY;
- });
+ StreamInfo currentStreamInfo = mStreamInfo;
+ mStreamInfo = streamInfo;
- ScheduledFuture<?> timeoutFuture =
- CameraXExecutors.myLooperExecutor().schedule(() -> {
- if (!surfaceUpdateFuture.isDone()) {
- surfaceUpdateCompleter.get().setException(new TimeoutException(
- "The surface isn't updated within: "
- + SURFACE_UPDATE_TIMEOUT));
- synchronized (mLock) {
- if (mSurfaceUpdateCompleter == surfaceUpdateCompleter.get()) {
- mSurfaceUpdateCompleter = null;
- }
- }
- }
- }, SURFACE_UPDATE_TIMEOUT, TimeUnit.MILLISECONDS);
-
- Futures.addCallback(surfaceUpdateFuture, new FutureCallback<Void>() {
- @Override
- public void onSuccess(@Nullable Void result) {
- getOutput().onSourceStateChanged(
- isStreamActive ? VideoOutput.SourceState.ACTIVE_STREAMING
- : VideoOutput.SourceState.ACTIVE_NON_STREAMING);
- timeoutFuture.cancel(true);
- }
-
- @Override
- public void onFailure(Throwable t) {
- Logger.d(TAG, "The surface update future didn't complete.", t);
- getOutput().onSourceStateChanged(
- isStreamActive ? VideoOutput.SourceState.ACTIVE_STREAMING
- : VideoOutput.SourceState.ACTIVE_NON_STREAMING);
- timeoutFuture.cancel(true);
- }
- }, CameraXExecutors.directExecutor());
-
- updateSessionConfig(mSessionConfigBuilder.build());
- notifyUpdated();
- }
- Integer currentStreamId = mStreamInfo.getId();
- if (!currentStreamId.equals(StreamInfo.STREAM_ID_ANY) && !currentStreamId.equals(
- streamInfo.getId())) {
+ // Doing resetPipeline() includes notifyReset/notifyUpdated(). Doing NotifyReset()
+ // includes notifyUpdated(). So we just take actions on higher order item for
+ // optimization.
+ if (!StreamInfo.NON_SURFACE_STREAM_ID.contains(currentStreamInfo.getId())
+ && !StreamInfo.NON_SURFACE_STREAM_ID.contains(streamInfo.getId())
+ && currentStreamInfo.getId() != streamInfo.getId()) {
// Reset pipeline if the stream ids are different, which means there's a new
// surface ready to be requested.
resetPipeline(getCameraId(), (VideoCaptureConfig<T>) getCurrentConfig(),
getAttachedSurfaceResolution());
- return;
+ } else if ((currentStreamInfo.getId() != STREAM_ID_ERROR
+ && streamInfo.getId() == STREAM_ID_ERROR)
+ || (currentStreamInfo.getId() == STREAM_ID_ERROR
+ && streamInfo.getId() != STREAM_ID_ERROR)) {
+ // If id switch to STREAM_ID_ERROR, it means VideoOutput is failed to setup video
+ // stream. The surface should be removed from camera. Vice versa.
+ applyStreamInfoToSessionConfigBuilder(mSessionConfigBuilder, streamInfo);
+ updateSessionConfig(mSessionConfigBuilder.build());
+ notifyReset();
+ } else if (currentStreamInfo.getStreamState() != streamInfo.getStreamState()) {
+ applyStreamInfoToSessionConfigBuilder(mSessionConfigBuilder, streamInfo);
+ updateSessionConfig(mSessionConfigBuilder.build());
+ notifyUpdated();
}
- mStreamInfo = streamInfo;
}
@Override
@@ -613,6 +562,87 @@
}
};
+ @UiThread
+ @SuppressWarnings("WeakerAccess") /* synthetic accessor */
+ void applyStreamInfoToSessionConfigBuilder(@NonNull SessionConfig.Builder sessionConfigBuilder,
+ @NonNull StreamInfo streamInfo) {
+ final boolean isStreamError = streamInfo.getId() == StreamInfo.STREAM_ID_ERROR;
+ final boolean isStreamActive = streamInfo.getStreamState() == StreamState.ACTIVE;
+ if (isStreamError && isStreamActive) {
+ throw new IllegalStateException(
+ "Unexpected stream state, stream is error but active");
+ }
+
+ sessionConfigBuilder.clearSurfaces();
+ if (!isStreamError) {
+ if (isStreamActive) {
+ sessionConfigBuilder.addSurface(mDeferrableSurface);
+ } else {
+ sessionConfigBuilder.addNonRepeatingSurface(mDeferrableSurface);
+ }
+ } else {
+ // Don't attach surface when stream is invalid.
+ }
+
+ setupSurfaceUpdateNotifier(sessionConfigBuilder, isStreamActive);
+ }
+
+ private void setupSurfaceUpdateNotifier(@NonNull SessionConfig.Builder sessionConfigBuilder,
+ boolean isStreamActive) {
+ AtomicReference<CallbackToFutureAdapter.Completer<Void>> surfaceUpdateCompleter =
+ new AtomicReference<>();
+ ListenableFuture<Void> surfaceUpdateFuture =
+ CallbackToFutureAdapter.getFuture(completer -> {
+ // Use the completer as the tag to identify the update.
+ sessionConfigBuilder.addTag(SURFACE_UPDATE_KEY, completer.hashCode());
+ synchronized (mLock) {
+ if (mSurfaceUpdateCompleter != null) {
+ // A newer update is issued before the previous update is
+ // completed. Fail the previous future.
+ mSurfaceUpdateCompleter.setException(new RuntimeException(
+ "A newer surface update is completed."));
+ }
+ mSurfaceUpdateCompleter = completer;
+ surfaceUpdateCompleter.set(completer);
+ }
+ return SURFACE_UPDATE_KEY;
+ });
+
+ ScheduledFuture<?> timeoutFuture =
+ CameraXExecutors.myLooperExecutor().schedule(() -> {
+ if (!surfaceUpdateFuture.isDone()) {
+ surfaceUpdateCompleter.get().setException(new TimeoutException(
+ "The surface isn't updated within: "
+ + SURFACE_UPDATE_TIMEOUT));
+ synchronized (mLock) {
+ if (mSurfaceUpdateCompleter == surfaceUpdateCompleter.get()) {
+ mSurfaceUpdateCompleter = null;
+ }
+ }
+ }
+ }, SURFACE_UPDATE_TIMEOUT, TimeUnit.MILLISECONDS);
+
+ Futures.addCallback(surfaceUpdateFuture, new FutureCallback<Void>() {
+ @Override
+ public void onSuccess(@Nullable Void result) {
+ onCompletion();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ Logger.d(TAG, "The surface update future didn't complete.", t);
+ onCompletion();
+ }
+
+ private void onCompletion() {
+ getOutput().onSourceStateChanged(
+ isStreamActive ? VideoOutput.SourceState.ACTIVE_STREAMING
+ : VideoOutput.SourceState.ACTIVE_NON_STREAMING);
+ timeoutFuture.cancel(true);
+ }
+ }, CameraXExecutors.directExecutor());
+ }
+
/**
* Set {@link ImageOutputConfig#OPTION_SUPPORTED_RESOLUTIONS} according to the resolution found
* by the {@link QualitySelector} in VideoOutput.
diff --git a/camera/camera-video/src/main/java/androidx/camera/video/internal/encoder/EncoderFactory.java b/camera/camera-video/src/main/java/androidx/camera/video/internal/encoder/EncoderFactory.java
new file mode 100644
index 0000000..829df86
--- /dev/null
+++ b/camera/camera-video/src/main/java/androidx/camera/video/internal/encoder/EncoderFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.camera.video.internal.encoder;
+
+import androidx.annotation.NonNull;
+
+import java.util.concurrent.Executor;
+
+/** Factory to create {@link Encoder} */
+public interface EncoderFactory {
+
+ /** Factory method to create {@link Encoder}. */
+ @NonNull
+ Encoder createEncoder(@NonNull Executor executor, @NonNull EncoderConfig encoderConfig)
+ throws InvalidConfigException;
+}