| // Copyright 2019 The Chromium Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/capture/video/chromeos/camera_app_device_impl.h" |
| |
| #include <algorithm> |
| #include <cmath> |
| |
| #include "base/task/bind_post_task.h" |
| #include "base/task/single_thread_task_runner.h" |
| #include "base/time/time.h" |
| #include "gpu/ipc/common/gpu_memory_buffer_impl.h" |
| #include "media/capture/video/chromeos/camera_app_device_bridge_impl.h" |
| #include "media/capture/video/chromeos/camera_device_context.h" |
| #include "media/capture/video/chromeos/camera_metadata_utils.h" |
| #include "third_party/libyuv/include/libyuv.h" |
| |
| namespace media { |
| |
| namespace { |
| |
| constexpr int kDetectionWidth = 256; |
| constexpr int kDetectionHeight = 256; |
| |
| } // namespace |
| |
| // static |
| int CameraAppDeviceImpl::GetPortraitSegResultCode( |
| const cros::mojom::CameraMetadataPtr* metadata) { |
| auto portrait_mode_segmentation_result = GetMetadataEntryAsSpan<uint8_t>( |
| *metadata, static_cast<cros::mojom::CameraMetadataTag>( |
| kPortraitModeSegmentationResultVendorKey)); |
| CHECK(!portrait_mode_segmentation_result.empty()); |
| return static_cast<int>(portrait_mode_segmentation_result[0]); |
| } |
| |
| CameraAppDeviceImpl::CameraAppDeviceImpl(const std::string& device_id) |
| : device_id_(device_id), |
| allow_new_ipc_weak_ptrs_(true), |
| capture_intent_(cros::mojom::CaptureIntent::kDefault), |
| camera_device_context_(nullptr) {} |
| |
| CameraAppDeviceImpl::~CameraAppDeviceImpl() { |
| // If the instance is bound, then this instance should only be destroyed when |
| // the mojo connection is dropped, which also happens on the mojo thread. |
| DCHECK(!mojo_task_runner_ || mojo_task_runner_->BelongsToCurrentThread()); |
| |
| // All the weak pointers of |weak_ptr_factory_| should be invalidated on |
| // camera device IPC thread before destroying CameraAppDeviceImpl. |
| DCHECK(!weak_ptr_factory_.HasWeakPtrs()); |
| } |
| |
| void CameraAppDeviceImpl::BindReceiver( |
| mojo::PendingReceiver<cros::mojom::CameraAppDevice> receiver) { |
| mojo_task_runner_ = base::SingleThreadTaskRunner::GetCurrentDefault(); |
| receivers_.Add(this, std::move(receiver)); |
| receivers_.set_disconnect_handler( |
| base::BindRepeating(&CameraAppDeviceImpl::OnMojoConnectionError, |
| weak_ptr_factory_for_mojo_.GetWeakPtr())); |
| document_scanner_service_ = ash::DocumentScannerServiceClient::Create(); |
| } |
| |
| base::WeakPtr<CameraAppDeviceImpl> CameraAppDeviceImpl::GetWeakPtr() { |
| return allow_new_ipc_weak_ptrs_ ? weak_ptr_factory_.GetWeakPtr() : nullptr; |
| } |
| |
| void CameraAppDeviceImpl::ResetOnDeviceIpcThread(base::OnceClosure callback, |
| bool should_disable_new_ptrs) { |
| if (should_disable_new_ptrs) { |
| allow_new_ipc_weak_ptrs_ = false; |
| } |
| weak_ptr_factory_.InvalidateWeakPtrs(); |
| std::move(callback).Run(); |
| } |
| |
| absl::optional<gfx::Range> CameraAppDeviceImpl::GetFpsRange() { |
| base::AutoLock lock(fps_ranges_lock_); |
| |
| return specified_fps_range_; |
| } |
| |
| gfx::Size CameraAppDeviceImpl::GetStillCaptureResolution() { |
| base::AutoLock lock(still_capture_resolution_lock_); |
| |
| return still_capture_resolution_; |
| } |
| |
| cros::mojom::CaptureIntent CameraAppDeviceImpl::GetCaptureIntent() { |
| base::AutoLock lock(capture_intent_lock_); |
| return capture_intent_; |
| } |
| |
| void CameraAppDeviceImpl::OnResultMetadataAvailable( |
| const cros::mojom::CameraMetadataPtr& metadata, |
| cros::mojom::StreamType streamType) { |
| mojo_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce(&CameraAppDeviceImpl::NotifyResultMetadataOnMojoThread, |
| weak_ptr_factory_for_mojo_.GetWeakPtr(), metadata.Clone(), |
| streamType)); |
| } |
| |
| void CameraAppDeviceImpl::OnShutterDone() { |
| mojo_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce(&CameraAppDeviceImpl::NotifyShutterDoneOnMojoThread, |
| weak_ptr_factory_for_mojo_.GetWeakPtr())); |
| } |
| |
| void CameraAppDeviceImpl::OnCameraInfoUpdated( |
| cros::mojom::CameraInfoPtr camera_info) { |
| base::AutoLock lock(camera_info_lock_); |
| camera_info_ = std::move(camera_info); |
| |
| if (!mojo_task_runner_) { |
| return; |
| } |
| mojo_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce(&CameraAppDeviceImpl::NotifyCameraInfoUpdatedOnMojoThread, |
| weak_ptr_factory_for_mojo_.GetWeakPtr())); |
| } |
| |
| void CameraAppDeviceImpl::SetCameraDeviceContext( |
| CameraDeviceContext* camera_device_context) { |
| base::AutoLock lock(camera_device_context_lock_); |
| camera_device_context_ = camera_device_context; |
| } |
| |
| void CameraAppDeviceImpl::MaybeDetectDocumentCorners( |
| std::unique_ptr<gpu::GpuMemoryBufferImpl> gmb, |
| VideoRotation rotation) { |
| if (!ash::DocumentScannerServiceClient::IsSupported()) { |
| return; |
| } |
| { |
| base::AutoLock lock(document_corners_observers_lock_); |
| if (document_corners_observers_.empty()) { |
| return; |
| } |
| } |
| mojo_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce(&CameraAppDeviceImpl::DetectDocumentCornersOnMojoThread, |
| weak_ptr_factory_for_mojo_.GetWeakPtr(), std::move(gmb), |
| rotation)); |
| } |
| |
| bool CameraAppDeviceImpl::IsMultipleStreamsEnabled() { |
| base::AutoLock lock(multi_stream_lock_); |
| return multi_stream_enabled_; |
| } |
| |
| void CameraAppDeviceImpl::TakePortraitModePhoto( |
| mojo::PendingRemote<cros::mojom::StillCaptureResultObserver> observer, |
| TakePortraitModePhotoCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| base::AutoLock lock(portrait_mode_callbacks_lock_); |
| portrait_mode_observers_.reset(); |
| portrait_mode_observers_.Bind(std::move(observer)); |
| take_portrait_photo_callbacks_.reset(); |
| |
| // Create two callbacks that will notify the client when the result is |
| // returned. The `normal_photo_callback` is for the normal photo, and |
| // `portrait_photo_callback` is for the portrait photo. |
| PortraitModeCallbacks take_portrait_photo_callbacks; |
| take_portrait_photo_callbacks.normal_photo_callback = |
| base::BindPostTaskToCurrentDefault( |
| base::BindOnce(&CameraAppDeviceImpl::NotifyPortraitResultOnMojoThread, |
| weak_ptr_factory_for_mojo_.GetWeakPtr(), |
| cros::mojom::Effect::kNoEffect)); |
| take_portrait_photo_callbacks.portrait_photo_callback = |
| base::BindPostTaskToCurrentDefault( |
| base::BindOnce(&CameraAppDeviceImpl::NotifyPortraitResultOnMojoThread, |
| weak_ptr_factory_for_mojo_.GetWeakPtr(), |
| cros::mojom::Effect::kPortraitMode)); |
| take_portrait_photo_callbacks_ = std::move(take_portrait_photo_callbacks); |
| |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::SetFpsRange(const gfx::Range& fps_range, |
| SetFpsRangeCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| const int entry_length = 2; |
| |
| base::AutoLock camera_info_lock(camera_info_lock_); |
| if (!camera_info_) { |
| LOG(ERROR) << "Camera info is still not available at this moment"; |
| std::move(callback).Run(false); |
| return; |
| } |
| auto& static_metadata = camera_info_->static_camera_characteristics; |
| auto available_fps_range_entries = GetMetadataEntryAsSpan<int32_t>( |
| static_metadata, cros::mojom::CameraMetadataTag:: |
| ANDROID_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES); |
| DCHECK(available_fps_range_entries.size() % entry_length == 0); |
| |
| bool is_valid = false; |
| int min_fps = static_cast<int>(fps_range.GetMin()); |
| int max_fps = static_cast<int>(fps_range.GetMax()); |
| for (size_t i = 0; i < available_fps_range_entries.size(); |
| i += entry_length) { |
| if (available_fps_range_entries[i] == min_fps && |
| available_fps_range_entries[i + 1] == max_fps) { |
| is_valid = true; |
| break; |
| } |
| } |
| |
| base::AutoLock lock(fps_ranges_lock_); |
| |
| if (is_valid) { |
| specified_fps_range_ = fps_range; |
| } else { |
| specified_fps_range_ = {}; |
| } |
| std::move(callback).Run(is_valid); |
| } |
| |
| void CameraAppDeviceImpl::SetStillCaptureResolution( |
| const gfx::Size& resolution, |
| SetStillCaptureResolutionCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| base::AutoLock lock(still_capture_resolution_lock_); |
| still_capture_resolution_ = resolution; |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::SetCaptureIntent( |
| cros::mojom::CaptureIntent capture_intent, |
| SetCaptureIntentCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| { |
| base::AutoLock lock(capture_intent_lock_); |
| capture_intent_ = capture_intent; |
| } |
| // Reset fps range for VCD to determine it if not explicitly set by app. |
| { |
| base::AutoLock lock(fps_ranges_lock_); |
| specified_fps_range_ = {}; |
| } |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::AddResultMetadataObserver( |
| mojo::PendingRemote<cros::mojom::ResultMetadataObserver> observer, |
| cros::mojom::StreamType stream_type, |
| AddResultMetadataObserverCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| stream_to_metadata_observers_map_[stream_type].Add(std::move(observer)); |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::AddCameraEventObserver( |
| mojo::PendingRemote<cros::mojom::CameraEventObserver> observer, |
| AddCameraEventObserverCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| camera_event_observers_.Add(std::move(observer)); |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::SetCameraFrameRotationEnabledAtSource( |
| bool is_enabled, |
| SetCameraFrameRotationEnabledAtSourceCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| bool is_success = false; |
| { |
| base::AutoLock lock(camera_device_context_lock_); |
| if (camera_device_context_) { |
| camera_device_context_->SetCameraFrameRotationEnabledAtSource(is_enabled); |
| is_success = true; |
| } |
| } |
| std::move(callback).Run(is_success); |
| } |
| |
| void CameraAppDeviceImpl::GetCameraFrameRotation( |
| GetCameraFrameRotationCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| uint32_t rotation = 0; |
| { |
| base::AutoLock lock(camera_device_context_lock_); |
| if (camera_device_context_ && |
| !camera_device_context_->IsCameraFrameRotationEnabledAtSource()) { |
| // The camera rotation value can only be [0, 90, 180, 270]. |
| rotation = static_cast<uint32_t>( |
| camera_device_context_->GetCameraFrameRotation()); |
| } |
| } |
| std::move(callback).Run(rotation); |
| } |
| |
| void CameraAppDeviceImpl::RegisterDocumentCornersObserver( |
| mojo::PendingRemote<cros::mojom::DocumentCornersObserver> observer, |
| RegisterDocumentCornersObserverCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| base::AutoLock lock(document_corners_observers_lock_); |
| document_corners_observers_.Add(std::move(observer)); |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::SetMultipleStreamsEnabled( |
| bool enabled, |
| SetMultipleStreamsEnabledCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| base::AutoLock lock(multi_stream_lock_); |
| multi_stream_enabled_ = enabled; |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::RegisterCameraInfoObserver( |
| mojo::PendingRemote<cros::mojom::CameraInfoObserver> observer, |
| RegisterCameraInfoObserverCallback callback) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| camera_info_observers_.Add(std::move(observer)); |
| std::move(callback).Run(); |
| |
| NotifyCameraInfoUpdatedOnMojoThread(); |
| } |
| |
| void CameraAppDeviceImpl::OnMojoConnectionError() { |
| CameraAppDeviceBridgeImpl::GetInstance()->OnDeviceMojoDisconnected( |
| device_id_); |
| } |
| |
| bool CameraAppDeviceImpl::IsCloseToPreviousDetectionRequest() { |
| return document_detection_timer_ && |
| document_detection_timer_->Elapsed().InMilliseconds() < 300; |
| } |
| |
| void CameraAppDeviceImpl::DetectDocumentCornersOnMojoThread( |
| std::unique_ptr<gpu::GpuMemoryBufferImpl> image, |
| VideoRotation rotation) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| DCHECK(document_scanner_service_); |
| |
| if (!document_scanner_service_->IsLoaded() || |
| IsCloseToPreviousDetectionRequest() || |
| has_ongoing_document_detection_task_) { |
| return; |
| } |
| |
| DCHECK(image); |
| if (!image->Map()) { |
| LOG(ERROR) << "Failed to map frame buffer"; |
| return; |
| } |
| auto frame_size = image->GetSize(); |
| int width = frame_size.width(); |
| int height = frame_size.height(); |
| |
| base::MappedReadOnlyRegion memory = base::ReadOnlySharedMemoryRegion::Create( |
| kDetectionWidth * kDetectionHeight * 3 / 2); |
| |
| auto* y_data = memory.mapping.GetMemoryAs<uint8_t>(); |
| auto* uv_data = y_data + kDetectionWidth * kDetectionHeight; |
| |
| int status = libyuv::NV12Scale( |
| static_cast<uint8_t*>(image->memory(0)), image->stride(0), |
| static_cast<uint8_t*>(image->memory(1)), image->stride(1), width, height, |
| y_data, kDetectionWidth, uv_data, kDetectionWidth, kDetectionWidth, |
| kDetectionHeight, libyuv::FilterMode::kFilterNone); |
| image->Unmap(); |
| if (status != 0) { |
| LOG(ERROR) << "Failed to scale buffer"; |
| return; |
| } |
| |
| has_ongoing_document_detection_task_ = true; |
| document_detection_timer_ = std::make_unique<base::ElapsedTimer>(); |
| // Since we destroy |document_scanner_service_| on mojo thread and this |
| // callback is also called on mojo thread, it should be safe to just use |
| // base::Unretained(this) here. |
| document_scanner_service_->DetectCornersFromNV12Image( |
| std::move(memory.region), |
| base::BindOnce( |
| &CameraAppDeviceImpl::OnDetectedDocumentCornersOnMojoThread, |
| base::Unretained(this), rotation)); |
| } |
| |
| void CameraAppDeviceImpl::OnDetectedDocumentCornersOnMojoThread( |
| VideoRotation rotation, |
| bool success, |
| const std::vector<gfx::PointF>& corners) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| has_ongoing_document_detection_task_ = false; |
| if (!success) { |
| LOG(ERROR) << "Failed to detect document corners"; |
| return; |
| } |
| |
| // Rotate a point in coordination space {x: [0.0, 1.0], y: [0.0, 1.0]} with |
| // anchor point {x: 0.5, y: 0.5}. |
| auto rotate_corner = [&](const gfx::PointF& corner) -> gfx::PointF { |
| float x = std::clamp(corner.x(), 0.0f, 1.0f); |
| float y = std::clamp(corner.y(), 0.0f, 1.0f); |
| |
| switch (rotation) { |
| case VIDEO_ROTATION_0: |
| return {x, y}; |
| case VIDEO_ROTATION_90: |
| return {1.0f - y, x}; |
| case VIDEO_ROTATION_180: |
| return {1.0f - x, 1.0f - y}; |
| case VIDEO_ROTATION_270: |
| return {y, 1.0f - x}; |
| default: |
| NOTREACHED(); |
| } |
| }; |
| |
| std::vector<gfx::PointF> rotated_corners; |
| for (auto& corner : corners) { |
| rotated_corners.push_back(rotate_corner(corner)); |
| } |
| |
| base::AutoLock lock(document_corners_observers_lock_); |
| for (auto& observer : document_corners_observers_) { |
| observer->OnDocumentCornersUpdated(rotated_corners); |
| } |
| } |
| |
| void CameraAppDeviceImpl::NotifyPortraitResultOnMojoThread( |
| cros::mojom::Effect effect, |
| const int32_t status, |
| media::mojom::BlobPtr blob) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| portrait_mode_observers_->OnStillCaptureDone(effect, status, std::move(blob)); |
| } |
| |
| void CameraAppDeviceImpl::NotifyShutterDoneOnMojoThread() { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| for (auto& observer : camera_event_observers_) { |
| observer->OnShutterDone(); |
| } |
| } |
| |
| void CameraAppDeviceImpl::NotifyResultMetadataOnMojoThread( |
| cros::mojom::CameraMetadataPtr metadata, |
| cros::mojom::StreamType streamType) { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| auto& metadata_observers = stream_to_metadata_observers_map_[streamType]; |
| for (auto& observer : metadata_observers) { |
| observer->OnMetadataAvailable(metadata.Clone()); |
| } |
| } |
| |
| void CameraAppDeviceImpl::NotifyCameraInfoUpdatedOnMojoThread() { |
| DCHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| base::AutoLock lock(camera_info_lock_); |
| if (!camera_info_) { |
| return; |
| } |
| for (auto& observer : camera_info_observers_) { |
| observer->OnCameraInfoUpdated(camera_info_.Clone()); |
| } |
| } |
| |
| absl::optional<PortraitModeCallbacks> |
| CameraAppDeviceImpl::ConsumePortraitModeCallbacks() { |
| base::AutoLock lock(portrait_mode_callbacks_lock_); |
| absl::optional<PortraitModeCallbacks> callbacks; |
| if (take_portrait_photo_callbacks_.has_value()) { |
| callbacks = std::move(take_portrait_photo_callbacks_); |
| take_portrait_photo_callbacks_.reset(); |
| } |
| return callbacks; |
| } |
| |
| void CameraAppDeviceImpl::SetCropRegion(const gfx::Rect& crop_region, |
| SetCropRegionCallback callback) { |
| CHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| base::AutoLock lock(crop_region_lock_); |
| crop_region_ = { |
| crop_region.x(), |
| crop_region.y(), |
| crop_region.width(), |
| crop_region.height(), |
| }; |
| |
| std::move(callback).Run(); |
| } |
| |
| void CameraAppDeviceImpl::ResetCropRegion(ResetCropRegionCallback callback) { |
| CHECK(mojo_task_runner_->BelongsToCurrentThread()); |
| |
| base::AutoLock lock(crop_region_lock_); |
| crop_region_.reset(); |
| |
| std::move(callback).Run(); |
| } |
| |
| std::optional<std::vector<int32_t>> CameraAppDeviceImpl::GetCropRegion() { |
| base::AutoLock lock(crop_region_lock_); |
| return crop_region_; |
| } |
| |
| } // namespace media |