[go: nahoru, domu]

blob: 0c0cf0959e2bbb32f3eb21e676571f917a8a4f7b [file] [log] [blame]
Franklin Wu30c76f62019-04-15 16:03:23 -07001/*
2 * Copyright 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package androidx.camera.extensions.impl;
17
charcoalcheneec786f52021-08-20 09:19:40 +080018import static android.hardware.camera2.CameraMetadata.CONTROL_AE_MODE_OFF;
19
Scott Nienb44f0772022-05-01 10:28:56 +080020import android.annotation.SuppressLint;
WenHung_Teng6a22abb2019-04-30 16:57:34 +080021import android.content.Context;
Franklin Wu30c76f62019-04-15 16:03:23 -070022import android.hardware.camera2.CameraCharacteristics;
23import android.hardware.camera2.CaptureRequest;
Scott Nienb44f0772022-05-01 10:28:56 +080024import android.hardware.camera2.CaptureResult;
Franklin Wuc84b18f2019-04-29 15:45:44 -070025import android.hardware.camera2.TotalCaptureResult;
Scott Nien76a0ba32023-08-11 16:10:22 +080026import android.hardware.camera2.params.SessionConfiguration;
Franklin Wu30c76f62019-04-15 16:03:23 -070027import android.media.Image;
28import android.media.ImageWriter;
29import android.os.Build;
30import android.util.Log;
Franklin Wuc84b18f2019-04-29 15:45:44 -070031import android.util.Pair;
Charcoal Chen4eeaeb22021-04-14 10:28:52 +080032import android.util.Range;
WenHung_Teng6a22abb2019-04-30 16:57:34 +080033import android.util.Size;
Franklin Wu30c76f62019-04-15 16:03:23 -070034import android.view.Surface;
35
TY Changd1057c82019-05-21 15:26:44 +080036import androidx.annotation.NonNull;
37import androidx.annotation.Nullable;
Trevor McGuiree0327b22021-09-22 22:59:57 -070038import androidx.annotation.RequiresApi;
TY Changd1057c82019-05-21 15:26:44 +080039
Franklin Wu30c76f62019-04-15 16:03:23 -070040import java.nio.ByteBuffer;
41import java.util.ArrayList;
Scott Nien73f4d382023-11-30 22:30:08 +080042import java.util.Collections;
Franklin Wu30c76f62019-04-15 16:03:23 -070043import java.util.List;
44import java.util.Map;
Scott Nienb44f0772022-05-01 10:28:56 +080045import java.util.concurrent.Executor;
Franklin Wu30c76f62019-04-15 16:03:23 -070046import java.util.concurrent.TimeUnit;
47
48/**
49 * Implementation for HDR image capture use case.
50 *
51 * <p>This class should be implemented by OEM and deployed to the target devices. 3P developers
52 * don't need to implement this, unless this is used for related testing usage.
Franklin Wu99529cb2019-10-07 11:35:44 -070053 *
54 * @since 1.0
Franklin Wu30c76f62019-04-15 16:03:23 -070055 */
Trevor McGuiree0327b22021-09-22 22:59:57 -070056@RequiresApi(21) // TODO(b/200306659): Remove and replace with annotation on package-info.java
Scott Nienb44f0772022-05-01 10:28:56 +080057@SuppressLint("UnknownNullness")
Franklin Wu30c76f62019-04-15 16:03:23 -070058public final class HdrImageCaptureExtenderImpl implements ImageCaptureExtenderImpl {
59 private static final String TAG = "HdrImageCaptureExtender";
60 private static final int UNDER_STAGE_ID = 0;
61 private static final int NORMAL_STAGE_ID = 1;
62 private static final int OVER_STAGE_ID = 2;
WenHung_Teng6a22abb2019-04-30 16:57:34 +080063 private static final int SESSION_STAGE_ID = 101;
charcoalcheneec786f52021-08-20 09:19:40 +080064 private static final long UNDER_EXPOSURE_TIME = TimeUnit.MILLISECONDS.toNanos(8);
65 private static final long NORMAL_EXPOSURE_TIME = TimeUnit.MILLISECONDS.toNanos(16);
66 private static final long OVER_EXPOSURE_TIME = TimeUnit.MILLISECONDS.toNanos(32);
Scott Nienc849f2a2023-10-27 14:39:56 +080067 private HdrImageCaptureExtenderCaptureProcessorImpl mCaptureProcessor = null;
Franklin Wu30c76f62019-04-15 16:03:23 -070068
69 public HdrImageCaptureExtenderImpl() {
70 }
71
72 @Override
Alan Viverette40eb9072022-06-22 14:24:52 -040073 public void init(@NonNull String cameraId,
74 @NonNull CameraCharacteristics cameraCharacteristics) {
Franklin Wudb1062c2019-04-23 18:22:35 -070075 }
Franklin Wu30c76f62019-04-15 16:03:23 -070076
77 @Override
TY Changd1057c82019-05-21 15:26:44 +080078 public boolean isExtensionAvailable(@NonNull String cameraId,
79 @Nullable CameraCharacteristics cameraCharacteristics) {
Charcoal Chen2f4582e2022-11-24 15:20:25 +080080 // Return false to skip tests since old devices do not support extensions.
81 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
charcoalcheneec786f52021-08-20 09:19:40 +080082 return false;
83 }
84
85 if (cameraCharacteristics == null) {
86 return false;
87 }
88
89 Range<Long> exposureTimeRange =
90 cameraCharacteristics.get(CameraCharacteristics.SENSOR_INFO_EXPOSURE_TIME_RANGE);
91
92 if (exposureTimeRange == null || !exposureTimeRange.contains(
93 Range.create(UNDER_EXPOSURE_TIME, OVER_EXPOSURE_TIME))) {
94 return false;
95 }
96
97 // The device needs to support CONTROL_AE_MODE_OFF for the testing CaptureStages
98 int[] availableAeModes =
99 cameraCharacteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_MODES);
100
101 if (availableAeModes != null) {
102 for (int mode : availableAeModes) {
103 if (mode == CONTROL_AE_MODE_OFF) {
104 return true;
105 }
106 }
107 }
108
109 return false;
Franklin Wu30c76f62019-04-15 16:03:23 -0700110 }
111
Scott Nien76a0ba32023-08-11 16:10:22 +0800112 @NonNull
Franklin Wu30c76f62019-04-15 16:03:23 -0700113 @Override
114 public List<CaptureStageImpl> getCaptureStages() {
115 // Under exposed capture stage
116 SettableCaptureStage captureStageUnder = new SettableCaptureStage(UNDER_STAGE_ID);
117 // Turn off AE so that ISO sensitivity can be controlled
118 captureStageUnder.addCaptureRequestParameters(CaptureRequest.CONTROL_AE_MODE,
119 CaptureRequest.CONTROL_AE_MODE_OFF);
120 captureStageUnder.addCaptureRequestParameters(CaptureRequest.SENSOR_EXPOSURE_TIME,
charcoalcheneec786f52021-08-20 09:19:40 +0800121 UNDER_EXPOSURE_TIME);
Franklin Wu30c76f62019-04-15 16:03:23 -0700122
123 // Normal exposed capture stage
124 SettableCaptureStage captureStageNormal = new SettableCaptureStage(NORMAL_STAGE_ID);
125 captureStageNormal.addCaptureRequestParameters(CaptureRequest.SENSOR_EXPOSURE_TIME,
charcoalcheneec786f52021-08-20 09:19:40 +0800126 NORMAL_EXPOSURE_TIME);
Franklin Wu30c76f62019-04-15 16:03:23 -0700127
128 // Over exposed capture stage
129 SettableCaptureStage captureStageOver = new SettableCaptureStage(OVER_STAGE_ID);
130 captureStageOver.addCaptureRequestParameters(CaptureRequest.SENSOR_EXPOSURE_TIME,
charcoalcheneec786f52021-08-20 09:19:40 +0800131 OVER_EXPOSURE_TIME);
Franklin Wu30c76f62019-04-15 16:03:23 -0700132
133 List<CaptureStageImpl> captureStages = new ArrayList<>();
134 captureStages.add(captureStageUnder);
135 captureStages.add(captureStageNormal);
136 captureStages.add(captureStageOver);
137 return captureStages;
138 }
139
Scott Nien76a0ba32023-08-11 16:10:22 +0800140
141 @Nullable
Franklin Wu30c76f62019-04-15 16:03:23 -0700142 @Override
143 public CaptureProcessorImpl getCaptureProcessor() {
Trevor McGuiree0327b22021-09-22 22:59:57 -0700144 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
Scott Nienc849f2a2023-10-27 14:39:56 +0800145 mCaptureProcessor = new HdrImageCaptureExtenderCaptureProcessorImpl();
146 return mCaptureProcessor;
Trevor McGuiree0327b22021-09-22 22:59:57 -0700147 } else {
148 return new NoOpCaptureProcessorImpl();
149 }
Franklin Wu30c76f62019-04-15 16:03:23 -0700150 }
WenHung_Teng6a22abb2019-04-30 16:57:34 +0800151
152 @Override
Alan Viverette40eb9072022-06-22 14:24:52 -0400153 public void onInit(@NonNull String cameraId,
154 @NonNull CameraCharacteristics cameraCharacteristics,
155 @NonNull Context context) {
WenHung_Teng6a22abb2019-04-30 16:57:34 +0800156 }
157
158 @Override
159 public void onDeInit() {
Scott Nienc849f2a2023-10-27 14:39:56 +0800160 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && mCaptureProcessor != null) {
161 mCaptureProcessor.release();
162 }
WenHung_Teng6a22abb2019-04-30 16:57:34 +0800163 }
164
Scott Nien76a0ba32023-08-11 16:10:22 +0800165 @Nullable
WenHung_Teng6a22abb2019-04-30 16:57:34 +0800166 @Override
167 public CaptureStageImpl onPresetSession() {
Charcoal Chen43a574e2021-03-19 14:27:05 +0800168 // The CaptureRequest parameters will be set via SessionConfiguration#setSessionParameters
169 // (CaptureRequest) which only supported from API level 28.
170 if (Build.VERSION.SDK_INT < 28) {
171 return null;
172 }
173
WenHung_Teng6a22abb2019-04-30 16:57:34 +0800174 SettableCaptureStage captureStage = new SettableCaptureStage(SESSION_STAGE_ID);
175 return captureStage;
176 }
177
Scott Nien76a0ba32023-08-11 16:10:22 +0800178 @Nullable
WenHung_Teng6a22abb2019-04-30 16:57:34 +0800179 @Override
180 public CaptureStageImpl onEnableSession() {
181 SettableCaptureStage captureStage = new SettableCaptureStage(SESSION_STAGE_ID);
182 return captureStage;
183 }
184
Scott Nien76a0ba32023-08-11 16:10:22 +0800185 @Nullable
WenHung_Teng6a22abb2019-04-30 16:57:34 +0800186 @Override
187 public CaptureStageImpl onDisableSession() {
188 SettableCaptureStage captureStage = new SettableCaptureStage(SESSION_STAGE_ID);
189 return captureStage;
190 }
191
192 @Override
193 public int getMaxCaptureStage() {
194 return 4;
195 }
196
Scott Nien76a0ba32023-08-11 16:10:22 +0800197 @Nullable
charcoalchen9429b072019-08-21 10:48:46 +0800198 @Override
199 public List<Pair<Integer, Size[]>> getSupportedResolutions() {
200 return null;
201 }
202
Charcoal Chen4eeaeb22021-04-14 10:28:52 +0800203 @Nullable
204 @Override
205 public Range<Long> getEstimatedCaptureLatencyRange(@Nullable Size captureOutputSize) {
206 return new Range<>(300L, 1000L);
207 }
Trevor McGuiree0327b22021-09-22 22:59:57 -0700208
209 @RequiresApi(23)
210 static final class HdrImageCaptureExtenderCaptureProcessorImpl implements CaptureProcessorImpl {
211 private ImageWriter mImageWriter;
Trevor McGuiree0327b22021-09-22 22:59:57 -0700212 @Override
Alan Viverette40eb9072022-06-22 14:24:52 -0400213 public void onOutputSurface(@NonNull Surface surface, int imageFormat) {
Scott Nienc849f2a2023-10-27 14:39:56 +0800214 mImageWriter = ImageWriter.newInstance(surface, 2);
Trevor McGuiree0327b22021-09-22 22:59:57 -0700215 }
216
217 @Override
Scott Nien76a0ba32023-08-11 16:10:22 +0800218 public void process(@NonNull Map<Integer, Pair<Image, TotalCaptureResult>> results) {
Scott Nienc849f2a2023-10-27 14:39:56 +0800219 process(results, null, null);
Scott Nien1b148222023-08-31 17:31:40 +0800220 }
221
222 @Override
223 public void process(@NonNull Map<Integer, Pair<Image, TotalCaptureResult>> results,
Scott Nienc849f2a2023-10-27 14:39:56 +0800224 @Nullable ProcessResultImpl resultCallback, @Nullable Executor executor) {
Trevor McGuiree0327b22021-09-22 22:59:57 -0700225 Log.d(TAG, "Started HDR CaptureProcessor");
226
Scott Nien1b148222023-08-31 17:31:40 +0800227 Executor executorForCallback = executor != null ? executor : (cmd) -> cmd.run();
228
Trevor McGuiree0327b22021-09-22 22:59:57 -0700229 // Check for availability of all requested images
230 if (!results.containsKey(UNDER_STAGE_ID)) {
231 Log.w(TAG,
232 "Unable to process since images does not contain "
233 + "underexposed image.");
234 return;
235 }
236
237 if (!results.containsKey(NORMAL_STAGE_ID)) {
238 Log.w(TAG,
239 "Unable to process since images does not contain normal "
240 + "exposed image.");
241 return;
242 }
243
244 if (!results.containsKey(OVER_STAGE_ID)) {
245 Log.w(TAG,
246 "Unable to process since images does not contain "
247 + "overexposed image.");
248 return;
249 }
250
251 // Do processing of images, our placeholder logic just copies the first
252 // Image into the output buffer.
253 List<Pair<Image, TotalCaptureResult>> imageDataPairs = new ArrayList<>(
254 results.values());
255 Image outputImage = mImageWriter.dequeueInputImage();
256
257 // Do processing here
258 // The sample here simply returns the normal image result
259 Image normalImage = imageDataPairs.get(NORMAL_STAGE_ID).first;
Trevor McGuiree0327b22021-09-22 22:59:57 -0700260 if (outputImage.getWidth() != normalImage.getWidth()
261 || outputImage.getHeight() != normalImage.getHeight()) {
262 throw new IllegalStateException(String.format("input image "
263 + "resolution [%d, %d] not the same as the "
264 + "output image[%d, %d]", normalImage.getWidth(),
265 normalImage.getHeight(), outputImage.getWidth(),
266 outputImage.getHeight()));
267 }
268
269 try {
270 // copy y plane
271 Image.Plane inYPlane = normalImage.getPlanes()[0];
272 Image.Plane outYPlane = outputImage.getPlanes()[0];
273 ByteBuffer inYBuffer = inYPlane.getBuffer();
274 ByteBuffer outYBuffer = outYPlane.getBuffer();
275 int inYPixelStride = inYPlane.getPixelStride();
276 int inYRowStride = inYPlane.getRowStride();
277 int outYPixelStride = outYPlane.getPixelStride();
278 int outYRowStride = outYPlane.getRowStride();
279 for (int x = 0; x < outputImage.getHeight(); x++) {
280 for (int y = 0; y < outputImage.getWidth(); y++) {
281 int inIndex = x * inYRowStride + y * inYPixelStride;
282 int outIndex = x * outYRowStride + y * outYPixelStride;
283 outYBuffer.put(outIndex, inYBuffer.get(inIndex));
284 }
285 }
Trevor McGuiree0327b22021-09-22 22:59:57 -0700286 // Copy UV
287 for (int i = 1; i < 3; i++) {
288 Image.Plane inPlane = normalImage.getPlanes()[i];
289 Image.Plane outPlane = outputImage.getPlanes()[i];
290 ByteBuffer inBuffer = inPlane.getBuffer();
291 ByteBuffer outBuffer = outPlane.getBuffer();
292 int inPixelStride = inPlane.getPixelStride();
293 int inRowStride = inPlane.getRowStride();
294 int outPixelStride = outPlane.getPixelStride();
295 int outRowStride = outPlane.getRowStride();
296 // UV are half width compared to Y
297 for (int x = 0; x < outputImage.getHeight() / 2; x++) {
298 for (int y = 0; y < outputImage.getWidth() / 2; y++) {
299 int inIndex = x * inRowStride + y * inPixelStride;
300 int outIndex = x * outRowStride + y * outPixelStride;
301 byte b = inBuffer.get(inIndex);
302 outBuffer.put(outIndex, b);
303 }
304 }
305 }
306 } catch (IllegalStateException e) {
307 Log.e(TAG, "Error accessing the Image: " + e);
308 // Since something went wrong, don't try to queue up the image.
309 // Instead let the Image writing get dropped.
310 return;
311 }
312
313 mImageWriter.queueInputImage(outputImage);
Scott Nien1b148222023-08-31 17:31:40 +0800314
315 TotalCaptureResult captureResult = results.get(NORMAL_STAGE_ID).second;
316
317 if (resultCallback != null) {
318 executorForCallback.execute(
319 () -> resultCallback.onCaptureCompleted(
320 captureResult.get(CaptureResult.SENSOR_TIMESTAMP),
321 getFilteredResults(captureResult)));
322 }
Trevor McGuiree0327b22021-09-22 22:59:57 -0700323
324 Log.d(TAG, "Completed HDR CaptureProcessor");
325 }
326
Scott Nien1b148222023-08-31 17:31:40 +0800327 @SuppressWarnings("unchecked")
328 private List<Pair<CaptureResult.Key, Object>> getFilteredResults(
329 TotalCaptureResult captureResult) {
330 List<Pair<CaptureResult.Key, Object>> list = new ArrayList<>();
331 for (CaptureResult.Key key : captureResult.getKeys()) {
332 list.add(new Pair<>(key, captureResult.get(key)));
333 }
Scott Nienb44f0772022-05-01 10:28:56 +0800334
Scott Nien1b148222023-08-31 17:31:40 +0800335 return list;
Scott Nienb44f0772022-05-01 10:28:56 +0800336 }
337
Scott Nien1b148222023-08-31 17:31:40 +0800338
Scott Nienb44f0772022-05-01 10:28:56 +0800339 @Override
Alan Viverette40eb9072022-06-22 14:24:52 -0400340 public void onResolutionUpdate(@NonNull Size size) {
Trevor McGuiree0327b22021-09-22 22:59:57 -0700341
342 }
343
344 @Override
345 public void onImageFormatUpdate(int imageFormat) {
346
347 }
Scott Nien76a0ba32023-08-11 16:10:22 +0800348
349 @Override
350 public void onPostviewOutputSurface(@NonNull Surface surface) {
Scott Nien76a0ba32023-08-11 16:10:22 +0800351 }
352
353 @Override
354 public void onResolutionUpdate(@NonNull Size size, @NonNull Size postviewSize) {
Scott Nien1b148222023-08-31 17:31:40 +0800355 onResolutionUpdate(size);
Scott Nien76a0ba32023-08-11 16:10:22 +0800356 }
357
358 @Override
359 public void processWithPostview(
360 @NonNull Map<Integer, Pair<Image, TotalCaptureResult>> results,
361 @NonNull ProcessResultImpl resultCallback, @Nullable Executor executor) {
Scott Nien1b148222023-08-31 17:31:40 +0800362 Log.d(TAG, "processWithPostview");
Scott Nienc849f2a2023-10-27 14:39:56 +0800363 process(results, resultCallback, executor);
364 }
365
366 public void release() {
367 if (mImageWriter != null) {
368 mImageWriter.close();
369 }
Scott Nien76a0ba32023-08-11 16:10:22 +0800370 }
Trevor McGuiree0327b22021-09-22 22:59:57 -0700371 }
Scott Nienb44f0772022-05-01 10:28:56 +0800372
Alan Viverette40eb9072022-06-22 14:24:52 -0400373 @NonNull
Scott Nienb44f0772022-05-01 10:28:56 +0800374 @Override
375 public List<CaptureRequest.Key> getAvailableCaptureRequestKeys() {
376 return null;
377 }
378
Scott Nien76a0ba32023-08-11 16:10:22 +0800379 @Override
380 public int onSessionType() {
381 return SessionConfiguration.SESSION_REGULAR;
382 }
383
384 @Nullable
385 @Override
386 public List<Pair<Integer, Size[]>> getSupportedPostviewResolutions(@NonNull Size captureSize) {
Scott Nien73f4d382023-11-30 22:30:08 +0800387 return Collections.emptyList();
Scott Nien76a0ba32023-08-11 16:10:22 +0800388 }
389
390 @Override
391 public boolean isCaptureProcessProgressAvailable() {
Scott Nienc849f2a2023-10-27 14:39:56 +0800392 return false;
Scott Nien76a0ba32023-08-11 16:10:22 +0800393 }
394
395 @Nullable
396 @Override
397 public Pair<Long, Long> getRealtimeCaptureLatency() {
398 return null;
399 }
400
401 @Override
402 public boolean isPostviewAvailable() {
Scott Nienc849f2a2023-10-27 14:39:56 +0800403 return false;
Scott Nien76a0ba32023-08-11 16:10:22 +0800404 }
405
Alan Viverette40eb9072022-06-22 14:24:52 -0400406 @NonNull
Scott Nienb44f0772022-05-01 10:28:56 +0800407 @Override
408 public List<CaptureResult.Key> getAvailableCaptureResultKeys() {
409 return null;
410 }
Franklin Wu30c76f62019-04-15 16:03:23 -0700411}