Rotate buffer in StreamSharing.
Previously, we let the virtual camera in StreamSharing inheriting the real camera (except for Preview). We did this to be consistent with the existing behavior. Now we realized that some image processing libraries handles the image transformation, such as MediaPipe. This means that CameraX needs to handles the transformation in the sharing step too. This will also make Preview consistent with other UseCases when acting as a child of StreamSharing.
In this CL, we update StreamSharing to always rotate the buffer for its children:
- In VirtualCameraAdapter, apply the rotation in the sharing node. This include the remaining rotation degrees from upstream node, plus the delta between the parent rotation and the child rotation if there is any.
Also fixed in this CL:
- In DefaultSurfaceProcessor, fixed a bug where we apply the GL flip and the texture transformation in the wrong order.
- Fixed a bug in EffectsFragment where the video cannot be recorded twice.
- Cleanup Surface in tests
Bug: 310260193
Test: manual test and ./gradlew bOS
Change-Id: I73565589b880261cb744224de32e4bdeceed60ce
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/processing/DefaultSurfaceProcessor.java b/camera/camera-core/src/main/java/androidx/camera/core/processing/DefaultSurfaceProcessor.java
index 94583c4..5477b9c 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/processing/DefaultSurfaceProcessor.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/processing/DefaultSurfaceProcessor.java
@@ -26,7 +26,6 @@
import android.graphics.Bitmap;
import android.graphics.ImageFormat;
import android.graphics.SurfaceTexture;
-import android.opengl.Matrix;
import android.os.Handler;
import android.os.HandlerThread;
import android.util.Size;
@@ -314,17 +313,13 @@
private Bitmap getBitmap(@NonNull Size size,
@NonNull float[] textureTransform,
int rotationDegrees) {
- float[] snapshotTransform = new float[16];
- Matrix.setIdentityM(snapshotTransform, 0);
-
- // Flip the snapshot. This is for reverting the GL transform added in SurfaceOutputImpl.
- MatrixExt.preVerticalFlip(snapshotTransform, 0.5f);
+ float[] snapshotTransform = textureTransform.clone();
// Rotate the output if requested.
MatrixExt.preRotate(snapshotTransform, rotationDegrees, 0.5f, 0.5f);
- // Apply the texture transform.
- Matrix.multiplyMM(snapshotTransform, 0, snapshotTransform, 0, textureTransform, 0);
+ // Flip the snapshot. This is for reverting the GL transform added in SurfaceOutputImpl.
+ MatrixExt.preVerticalFlip(snapshotTransform, 0.5f);
// Update the size based on the rotation degrees.
size = rotateSize(size, rotationDegrees);
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/StreamSharing.java b/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/StreamSharing.java
index aff1c4f..0f2c246 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/StreamSharing.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/StreamSharing.java
@@ -256,7 +256,8 @@
// Transform the input based on virtual camera configuration.
Map<UseCase, SurfaceProcessorNode.OutConfig> outConfigMap =
- mVirtualCameraAdapter.getChildrenOutConfigs(mSharingInputEdge);
+ mVirtualCameraAdapter.getChildrenOutConfigs(mSharingInputEdge,
+ getTargetRotationInternal());
SurfaceProcessorNode.Out out = mSharingNode.transform(
SurfaceProcessorNode.In.of(mSharingInputEdge,
new ArrayList<>(outConfigMap.values())));
diff --git a/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/VirtualCameraAdapter.java b/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/VirtualCameraAdapter.java
index 22066a5..6323d6c 100644
--- a/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/VirtualCameraAdapter.java
+++ b/camera/camera-core/src/main/java/androidx/camera/core/streamsharing/VirtualCameraAdapter.java
@@ -27,6 +27,7 @@
import static androidx.camera.core.impl.utils.Threads.checkMainThread;
import static androidx.camera.core.impl.utils.TransformUtils.getRotatedSize;
import static androidx.camera.core.impl.utils.TransformUtils.rectToSize;
+import static androidx.camera.core.impl.utils.TransformUtils.within360;
import static androidx.camera.core.streamsharing.DynamicRangeUtils.resolveDynamicRange;
import static androidx.camera.core.streamsharing.ResolutionUtils.getMergedResolutions;
import static androidx.core.util.Preconditions.checkState;
@@ -36,6 +37,7 @@
import android.graphics.ImageFormat;
import android.os.Build;
import android.util.Size;
+import android.view.Surface;
import androidx.annotation.IntRange;
import androidx.annotation.MainThread;
@@ -52,6 +54,7 @@
import androidx.camera.core.impl.CameraCaptureResult;
import androidx.camera.core.impl.CameraInternal;
import androidx.camera.core.impl.DeferrableSurface;
+import androidx.camera.core.impl.ImageOutputConfig;
import androidx.camera.core.impl.MutableConfig;
import androidx.camera.core.impl.SessionConfig;
import androidx.camera.core.impl.UseCaseConfig;
@@ -208,21 +211,26 @@
* Gets {@link OutConfig} for children {@link UseCase} based on the input edge.
*/
@NonNull
- Map<UseCase, OutConfig> getChildrenOutConfigs(@NonNull SurfaceEdge cameraEdge) {
+ Map<UseCase, OutConfig> getChildrenOutConfigs(@NonNull SurfaceEdge cameraEdge,
+ @ImageOutputConfig.RotationValue int parentTargetRotation) {
Map<UseCase, OutConfig> outConfigs = new HashMap<>();
+ int parentRotationDegrees = mParentCamera.getCameraInfo().getSensorRotationDegrees(
+ parentTargetRotation);
for (UseCase useCase : mChildren) {
// TODO(b/264936115): This is a temporary solution where children use the parent
// stream without changing it. Later we will update it to allow
// cropping/down-sampling to better match children UseCase config.
- int rotationDegrees = getChildRotationDegrees(useCase);
+ int childRotationDegrees = getChildRotationDegrees(useCase);
requireNonNull(mChildrenVirtualCameras.get(useCase))
- .setRotationDegrees(rotationDegrees);
+ .setRotationDegrees(childRotationDegrees);
+ int childParentDelta = within360(
+ cameraEdge.getRotationDegrees() + childRotationDegrees - parentRotationDegrees);
outConfigs.put(useCase, OutConfig.of(
getChildTargetType(useCase),
getChildFormat(useCase),
cameraEdge.getCropRect(),
- getRotatedSize(cameraEdge.getCropRect(), rotationDegrees),
- rotationDegrees,
+ getRotatedSize(cameraEdge.getCropRect(), childParentDelta),
+ childParentDelta,
useCase.isMirroringRequired(mParentCamera)));
}
return outConfigs;
@@ -329,13 +337,10 @@
@IntRange(from = 0, to = 359)
private int getChildRotationDegrees(@NonNull UseCase child) {
- if (child instanceof Preview) {
- // Rotate the buffer for Preview because SurfaceView cannot handle rotation.
- return mParentCamera.getCameraInfo().getSensorRotationDegrees(
- ((Preview) child).getTargetRotation());
- }
- // By default, sharing node does not rotate
- return 0;
+ int childTargetRotation = ((ImageOutputConfig) child.getCurrentConfig())
+ .getTargetRotation(Surface.ROTATION_0);
+ return mParentCamera.getCameraInfo().getSensorRotationDegrees(
+ childTargetRotation);
}
private static int getChildFormat(@NonNull UseCase useCase) {
diff --git a/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/StreamSharingTest.kt b/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/StreamSharingTest.kt
index 6d14cf7..f910404 100644
--- a/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/StreamSharingTest.kt
+++ b/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/StreamSharingTest.kt
@@ -136,6 +136,8 @@
streamSharing.unbindFromCamera(streamSharing.camera!!)
}
effectProcessor.release()
+ sharingProcessor.cleanUp()
+ effectProcessor.cleanUp()
shadowOf(getMainLooper()).idle()
}
@@ -220,12 +222,12 @@
fun childTakingPicture_getJpegQuality() {
// Arrange: set up StreamSharing with min latency ImageCapture as child
val imageCapture = ImageCapture.Builder()
- .setTargetRotation(Surface.ROTATION_90)
.setCaptureMode(CAPTURE_MODE_MINIMIZE_LATENCY)
.build()
streamSharing = StreamSharing(camera, setOf(child1, imageCapture), useCaseConfigFactory)
streamSharing.bindToCamera(camera, null, defaultConfig)
streamSharing.onSuggestedStreamSpecUpdated(StreamSpec.builder(size).build())
+ imageCapture.targetRotation = Surface.ROTATION_90
// Act: the child takes a picture.
imageCapture.takePicture(directExecutor(), object : ImageCapture.OnImageCapturedCallback() {
@@ -526,6 +528,17 @@
}
@Test
+ fun bindChildToCamera_virtualCameraHasNoRotationDegrees() {
+ // Act.
+ streamSharing.bindToCamera(frontCamera, null, null)
+ // Assert.
+ assertThat(child1.camera!!.cameraInfoInternal.getSensorRotationDegrees(Surface.ROTATION_0))
+ .isEqualTo(0)
+ assertThat(child2.camera!!.cameraInfoInternal.getSensorRotationDegrees(Surface.ROTATION_0))
+ .isEqualTo(0)
+ }
+
+ @Test
fun bindAndUnbindParent_propagatesToChildren() {
// Assert: children not bound to camera by default.
assertThat(child1.camera).isNull()
diff --git a/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/VirtualCameraAdapterTest.kt b/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/VirtualCameraAdapterTest.kt
index dfcb2798..9c85fe8 100644
--- a/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/VirtualCameraAdapterTest.kt
+++ b/camera/camera-core/src/test/java/androidx/camera/core/streamsharing/VirtualCameraAdapterTest.kt
@@ -155,6 +155,8 @@
assertThat(getUseCaseSurface(preview)).isNotNull()
// Cleanup.
preview.unbindFromCamera(parentCamera)
+ surfaceTexture.release()
+ surface.release()
}
private fun getUseCaseSurface(useCase: UseCase): DeferrableSurface? {
@@ -251,7 +253,7 @@
// Arrange.
val cropRect = Rect(10, 10, 410, 310)
val preview = Preview.Builder().setTargetRotation(Surface.ROTATION_90).build()
- val imageCapture = ImageCapture.Builder().build()
+ val imageCapture = ImageCapture.Builder().setTargetRotation(Surface.ROTATION_0).build()
adapter = VirtualCameraAdapter(
parentCamera, setOf(preview, child2, imageCapture), useCaseConfigFactory
) { _, _ ->
@@ -260,7 +262,8 @@
// Act.
val outConfigs = adapter.getChildrenOutConfigs(
- createSurfaceEdge(cropRect = cropRect)
+ createSurfaceEdge(cropRect = cropRect, rotationDegrees = 90),
+ Surface.ROTATION_90
)
// Assert: preview config
@@ -268,19 +271,23 @@
assertThat(previewOutConfig.format).isEqualTo(INTERNAL_DEFINED_IMAGE_FORMAT_PRIVATE)
assertThat(previewOutConfig.targets).isEqualTo(PREVIEW)
assertThat(previewOutConfig.cropRect).isEqualTo(cropRect)
+ // Preview's target rotation matches the parent's, so it only applies the 90° rotation.
assertThat(previewOutConfig.size).isEqualTo(Size(300, 400))
- assertThat(previewOutConfig.rotationDegrees).isEqualTo(270)
+ assertThat(previewOutConfig.rotationDegrees).isEqualTo(90)
assertThat(previewOutConfig.mirroring).isFalse()
// Assert: ImageCapture config
val imageOutConfig = outConfigs[imageCapture]!!
assertThat(imageOutConfig.format).isEqualTo(ImageFormat.JPEG)
assertThat(imageOutConfig.targets).isEqualTo(IMAGE_CAPTURE)
+ // ImageCapture's target rotation does not match the parent's, so it applies the delta on
+ // top of the 90° rotation.
+ assertThat(imageOutConfig.size).isEqualTo(Size(400, 300))
+ assertThat(imageOutConfig.rotationDegrees).isEqualTo(180)
// Assert: child2
val outConfig2 = outConfigs[child2]!!
assertThat(outConfig2.format).isEqualTo(INTERNAL_DEFINED_IMAGE_FORMAT_PRIVATE)
assertThat(outConfig2.targets).isEqualTo(VIDEO_CAPTURE)
assertThat(outConfig2.cropRect).isEqualTo(cropRect)
- assertThat(outConfig2.size).isEqualTo(Size(400, 300))
assertThat(outConfig2.mirroring).isTrue()
}
diff --git a/camera/camera-testing/src/main/java/androidx/camera/testing/impl/fakes/FakeSurfaceProcessor.java b/camera/camera-testing/src/main/java/androidx/camera/testing/impl/fakes/FakeSurfaceProcessor.java
index c0aed34..d97e4fe 100644
--- a/camera/camera-testing/src/main/java/androidx/camera/testing/impl/fakes/FakeSurfaceProcessor.java
+++ b/camera/camera-testing/src/main/java/androidx/camera/testing/impl/fakes/FakeSurfaceProcessor.java
@@ -43,7 +43,6 @@
private final Executor mExecutor;
private final boolean mAutoCloseSurfaceOutput;
-
@Nullable
private SurfaceRequest mSurfaceRequest;
@NonNull
diff --git a/camera/integration-tests/viewtestapp/src/main/java/androidx/camera/integration/view/EffectsFragment.kt b/camera/integration-tests/viewtestapp/src/main/java/androidx/camera/integration/view/EffectsFragment.kt
index 1d811a0..ca25ba9 100644
--- a/camera/integration-tests/viewtestapp/src/main/java/androidx/camera/integration/view/EffectsFragment.kt
+++ b/camera/integration-tests/viewtestapp/src/main/java/androidx/camera/integration/view/EffectsFragment.kt
@@ -217,6 +217,7 @@
private fun stopRecording() {
record.text = "Record"
recording?.stop()
+ recording = null
}
private fun getNewVideoOutputMediaStoreOptions(): MediaStoreOutputOptions {