| // Copyright 2020 The Chromium Authors |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/video/video_encode_accelerator_adapter.h" |
| |
| #include <limits> |
| #include <vector> |
| |
| #include "base/functional/callback_helpers.h" |
| #include "base/logging.h" |
| #include "base/memory/ref_counted.h" |
| #include "base/numerics/checked_math.h" |
| #include "base/numerics/safe_conversions.h" |
| #include "base/synchronization/waitable_event.h" |
| #include "base/task/bind_post_task.h" |
| #include "base/task/sequenced_task_runner.h" |
| #include "base/time/time.h" |
| #include "base/trace_event/trace_event.h" |
| #include "build/build_config.h" |
| #include "gpu/ipc/common/gpu_memory_buffer_support.h" |
| #include "media/base/bitstream_buffer.h" |
| #include "media/base/media_log.h" |
| #include "media/base/media_switches.h" |
| #include "media/base/svc_scalability_mode.h" |
| #include "media/base/video_frame.h" |
| #include "media/base/video_util.h" |
| #if BUILDFLAG(USE_PROPRIETARY_CODECS) |
| #include "media/formats/mp4/h264_annex_b_to_avc_bitstream_converter.h" |
| #if BUILDFLAG(ENABLE_PLATFORM_HEVC) && \ |
| BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| #include "media/formats/mp4/h265_annex_b_to_hevc_bitstream_converter.h" |
| #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC) && |
| // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) |
| #include "media/video/gpu_video_accelerator_factories.h" |
| #include "media/video/video_encoder_info.h" |
| |
| namespace media { |
| |
| namespace { |
| |
| // HW encoders expect a nonzero bitrate, so |kVEADefaultBitratePerPixel| is used |
| // to estimate bits per second for ~30 fps with ~1/16 compression rate. |
| constexpr int kVEADefaultBitratePerPixel = 2; |
| |
| uint32_t ComputeCheckedDefaultBitrate(const gfx::Size& frame_size) { |
| base::CheckedNumeric<uint32_t> checked_bitrate_product = |
| base::CheckMul<uint32_t>(frame_size.width(), frame_size.height(), |
| kVEADefaultBitratePerPixel); |
| // If the product has overflowed, clamp it to uint32_t max |
| return checked_bitrate_product.ValueOrDefault( |
| std::numeric_limits<uint32_t>::max()); |
| } |
| |
| uint32_t ComputeCheckedPeakBitrate(uint32_t target_bitrate) { |
| // TODO(crbug.com/1342850): Reconsider whether this is good peak bps. |
| base::CheckedNumeric<uint32_t> checked_bitrate_product = |
| base::CheckMul<uint32_t>(target_bitrate, 10u); |
| return checked_bitrate_product.ValueOrDefault( |
| std::numeric_limits<uint32_t>::max()); |
| } |
| |
| Bitrate CreateBitrate( |
| const std::optional<Bitrate>& requested_bitrate, |
| const gfx::Size& frame_size, |
| VideoEncodeAccelerator::SupportedRateControlMode supported_rc_modes) { |
| uint32_t default_bitrate = ComputeCheckedDefaultBitrate(frame_size); |
| if (supported_rc_modes & VideoEncodeAccelerator::kVariableMode) { |
| // VEA supports VBR. Use |requested_bitrate| or VBR if bitrate is not |
| // specified. |
| return requested_bitrate.value_or(Bitrate::VariableBitrate( |
| default_bitrate, ComputeCheckedPeakBitrate(default_bitrate))); |
| } |
| // VEA doesn't support VBR. The bitrate configured to VEA must be CBR. In |
| // other words, if |requested_bitrate| is CBR, bitrate mode fallbacks to VBR. |
| if (requested_bitrate && |
| requested_bitrate->mode() == Bitrate::Mode::kConstant) { |
| return *requested_bitrate; |
| } |
| |
| return Bitrate::ConstantBitrate( |
| requested_bitrate ? requested_bitrate->target_bps() : default_bitrate); |
| } |
| |
| VideoEncodeAccelerator::Config SetUpVeaConfig( |
| VideoCodecProfile profile, |
| const VideoEncoder::Options& opts, |
| VideoPixelFormat format, |
| VideoFrame::StorageType storage_type, |
| VideoEncodeAccelerator::SupportedRateControlMode supported_rc_modes, |
| VideoEncodeAccelerator::Config::EncoderType required_encoder_type) { |
| std::optional<uint32_t> initial_framerate; |
| if (opts.framerate.has_value()) |
| initial_framerate = static_cast<uint32_t>(opts.framerate.value()); |
| |
| Bitrate bitrate = |
| CreateBitrate(opts.bitrate, opts.frame_size, supported_rc_modes); |
| auto config = |
| VideoEncodeAccelerator::Config(format, opts.frame_size, profile, bitrate); |
| config.initial_framerate = initial_framerate; |
| config.gop_length = opts.keyframe_interval; |
| |
| if (opts.content_hint) { |
| switch (*opts.content_hint) { |
| case media::VideoEncoder::ContentHint::Camera: |
| config.content_type = |
| VideoEncodeAccelerator::Config::ContentType::kCamera; |
| break; |
| case media::VideoEncoder::ContentHint::Screen: |
| config.content_type = |
| VideoEncodeAccelerator::Config::ContentType::kDisplay; |
| break; |
| } |
| } |
| |
| if (opts.latency_mode == VideoEncoder::LatencyMode::Realtime) { |
| config.drop_frame_thresh_percentage = |
| GetDefaultVideoEncoderDropFrameThreshold(); |
| } |
| |
| size_t num_temporal_layers = 1; |
| if (opts.scalability_mode) { |
| switch (opts.scalability_mode.value()) { |
| case SVCScalabilityMode::kL1T1: |
| // Nothing to do |
| break; |
| case SVCScalabilityMode::kL1T2: |
| num_temporal_layers = 2; |
| break; |
| case SVCScalabilityMode::kL1T3: |
| num_temporal_layers = 3; |
| break; |
| default: |
| NOTREACHED() << "Unsupported SVC: " |
| << GetScalabilityModeName(opts.scalability_mode.value()); |
| } |
| } |
| if (num_temporal_layers > 1) { |
| VideoEncodeAccelerator::Config::SpatialLayer layer; |
| layer.width = opts.frame_size.width(); |
| layer.height = opts.frame_size.height(); |
| layer.bitrate_bps = config.bitrate.target_bps(); |
| if (initial_framerate.has_value()) |
| layer.framerate = initial_framerate.value(); |
| layer.num_of_temporal_layers = num_temporal_layers; |
| config.spatial_layers.push_back(layer); |
| } |
| |
| config.require_low_delay = |
| opts.latency_mode == VideoEncoder::LatencyMode::Realtime; |
| config.required_encoder_type = required_encoder_type; |
| |
| const bool is_rgb = |
| format == PIXEL_FORMAT_XBGR || format == PIXEL_FORMAT_XRGB || |
| format == PIXEL_FORMAT_ABGR || format == PIXEL_FORMAT_ARGB; |
| |
| // Override the provided format if incoming frames are RGB -- they'll be |
| // converted to I420 or NV12 depending on the VEA configuration. |
| if (is_rgb) |
| config.input_format = PIXEL_FORMAT_I420; |
| |
| #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) |
| if (format != PIXEL_FORMAT_I420 || |
| !VideoFrame::IsStorageTypeMappable(storage_type)) { |
| // ChromeOS/Linux hardware video encoders supports I420 on-memory |
| // VideoFrame and NV12 GpuMemoryBuffer VideoFrame. |
| // For other VideoFrames than them, some processing e.g. format conversion |
| // is required. Let the destination buffer be GpuMemoryBuffer because a |
| // hardware encoder can process it more efficiently than on-memory buffer. |
| config.input_format = PIXEL_FORMAT_NV12; |
| config.storage_type = |
| VideoEncodeAccelerator::Config::StorageType::kGpuMemoryBuffer; |
| } |
| #endif |
| |
| return config; |
| } |
| |
| } // namespace |
| |
| class VideoEncodeAcceleratorAdapter::GpuMemoryBufferVideoFramePool |
| : public base::RefCountedThreadSafe<GpuMemoryBufferVideoFramePool> { |
| public: |
| GpuMemoryBufferVideoFramePool(GpuVideoAcceleratorFactories* gpu_factories, |
| const gfx::Size& coded_size) |
| : gpu_factories_(gpu_factories), coded_size_(coded_size) {} |
| GpuMemoryBufferVideoFramePool(const GpuMemoryBufferVideoFramePool&) = delete; |
| GpuMemoryBufferVideoFramePool& operator=( |
| const GpuMemoryBufferVideoFramePool&) = delete; |
| |
| scoped_refptr<VideoFrame> MaybeCreateVideoFrame( |
| const gfx::Size& visible_size) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| DCHECK(gfx::Rect(coded_size_).Contains(gfx::Rect(visible_size))); |
| |
| if (available_gmbs_.empty()) { |
| constexpr auto kBufferFormat = gfx::BufferFormat::YUV_420_BIPLANAR; |
| constexpr auto kBufferUsage = |
| gfx::BufferUsage::VEA_READ_CAMERA_AND_CPU_READ_WRITE; |
| auto gmb = gpu_factories_->CreateGpuMemoryBuffer( |
| coded_size_, kBufferFormat, kBufferUsage); |
| if (!gmb) |
| return nullptr; |
| |
| available_gmbs_.push_back(std::move(gmb)); |
| } |
| |
| auto gmb = std::move(available_gmbs_.back()); |
| available_gmbs_.pop_back(); |
| |
| VideoFrame::ReleaseMailboxAndGpuMemoryBufferCB reuse_cb = |
| base::BindPostTaskToCurrentDefault( |
| base::BindOnce(&GpuMemoryBufferVideoFramePool::ReuseFrame, this)); |
| const gpu::MailboxHolder kEmptyMailBoxes[media::VideoFrame::kMaxPlanes] = |
| {}; |
| return VideoFrame::WrapExternalGpuMemoryBuffer( |
| gfx::Rect(visible_size), visible_size, std::move(gmb), kEmptyMailBoxes, |
| std::move(reuse_cb), base::TimeDelta()); |
| } |
| |
| private: |
| friend class RefCountedThreadSafe<GpuMemoryBufferVideoFramePool>; |
| ~GpuMemoryBufferVideoFramePool() = default; |
| |
| void ReuseFrame(const gpu::SyncToken& token, |
| std::unique_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| constexpr size_t kMaxPooledFrames = 5; |
| if (available_gmbs_.size() < kMaxPooledFrames) |
| available_gmbs_.push_back(std::move(gpu_memory_buffer)); |
| } |
| |
| const raw_ptr<GpuVideoAcceleratorFactories> gpu_factories_; |
| const gfx::Size coded_size_; |
| |
| std::vector<std::unique_ptr<gfx::GpuMemoryBuffer>> available_gmbs_; |
| |
| SEQUENCE_CHECKER(sequence_checker_); |
| }; |
| |
| class VideoEncodeAcceleratorAdapter::ReadOnlyRegionPool |
| : public base::RefCountedThreadSafe<ReadOnlyRegionPool> { |
| public: |
| struct Handle { |
| using ReuseBufferCallback = |
| base::OnceCallback<void(std::unique_ptr<base::MappedReadOnlyRegion>)>; |
| Handle(std::unique_ptr<base::MappedReadOnlyRegion> mapped_region, |
| ReuseBufferCallback reuse_buffer_cb) |
| : owned_mapped_region(std::move(mapped_region)), |
| reuse_buffer_cb(std::move(reuse_buffer_cb)) { |
| DCHECK(owned_mapped_region); |
| } |
| |
| ~Handle() { |
| if (reuse_buffer_cb) { |
| DCHECK(owned_mapped_region); |
| std::move(reuse_buffer_cb).Run(std::move(owned_mapped_region)); |
| } |
| } |
| |
| Handle(const Handle&) = delete; |
| Handle& operator=(const Handle&) = delete; |
| |
| bool IsValid() const { |
| return owned_mapped_region && owned_mapped_region->IsValid(); |
| } |
| const base::ReadOnlySharedMemoryRegion* region() const { |
| DCHECK(IsValid()); |
| return &owned_mapped_region->region; |
| } |
| const base::WritableSharedMemoryMapping* mapping() const { |
| DCHECK(IsValid()); |
| return &owned_mapped_region->mapping; |
| } |
| |
| private: |
| std::unique_ptr<base::MappedReadOnlyRegion> owned_mapped_region; |
| ReuseBufferCallback reuse_buffer_cb; |
| }; |
| |
| explicit ReadOnlyRegionPool(size_t buffer_size) : buffer_size_(buffer_size) {} |
| ReadOnlyRegionPool(const ReadOnlyRegionPool&) = delete; |
| ReadOnlyRegionPool& operator=(const ReadOnlyRegionPool&) = delete; |
| |
| std::unique_ptr<Handle> MaybeAllocateBuffer() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| if (available_buffers_.empty()) { |
| available_buffers_.push_back(std::make_unique<base::MappedReadOnlyRegion>( |
| base::ReadOnlySharedMemoryRegion::Create(buffer_size_))); |
| if (!available_buffers_.back()->IsValid()) { |
| available_buffers_.pop_back(); |
| return nullptr; |
| } |
| } |
| |
| auto mapped_region = std::move(available_buffers_.back()); |
| available_buffers_.pop_back(); |
| DCHECK(mapped_region->IsValid()); |
| |
| return std::make_unique<Handle>( |
| std::move(mapped_region), |
| base::BindPostTaskToCurrentDefault( |
| base::BindOnce(&ReadOnlyRegionPool::ReuseBuffer, this))); |
| } |
| |
| private: |
| friend class RefCountedThreadSafe<ReadOnlyRegionPool>; |
| ~ReadOnlyRegionPool() = default; |
| |
| void ReuseBuffer(std::unique_ptr<base::MappedReadOnlyRegion> region) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); |
| constexpr size_t kMaxPooledBuffers = 5; |
| if (available_buffers_.size() < kMaxPooledBuffers) |
| available_buffers_.push_back(std::move(region)); |
| } |
| |
| const size_t buffer_size_; |
| std::vector<std::unique_ptr<base::MappedReadOnlyRegion>> available_buffers_; |
| |
| SEQUENCE_CHECKER(sequence_checker_); |
| }; |
| |
| VideoEncodeAcceleratorAdapter::PendingOp::PendingOp() = default; |
| VideoEncodeAcceleratorAdapter::PendingOp::~PendingOp() = default; |
| |
| VideoEncodeAcceleratorAdapter::VideoEncodeAcceleratorAdapter( |
| GpuVideoAcceleratorFactories* gpu_factories, |
| std::unique_ptr<MediaLog> media_log, |
| scoped_refptr<base::SequencedTaskRunner> callback_task_runner, |
| VideoEncodeAccelerator::Config::EncoderType required_encoder_type) |
| : output_pool_(base::MakeRefCounted<base::UnsafeSharedMemoryPool>()), |
| gpu_factories_(gpu_factories), |
| media_log_(std::move(media_log)), |
| accelerator_task_runner_(gpu_factories_->GetTaskRunner()), |
| callback_task_runner_(std::move(callback_task_runner)), |
| required_encoder_type_(required_encoder_type) { |
| DETACH_FROM_SEQUENCE(accelerator_sequence_checker_); |
| } |
| |
| VideoEncodeAcceleratorAdapter::~VideoEncodeAcceleratorAdapter() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| output_pool_->Shutdown(); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::DestroyAsync( |
| std::unique_ptr<VideoEncodeAcceleratorAdapter> self) { |
| DCHECK(self); |
| auto runner = self->accelerator_task_runner_; |
| DCHECK(runner); |
| if (!runner->RunsTasksInCurrentSequence()) |
| runner->DeleteSoon(FROM_HERE, std::move(self)); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::SetInputBufferPreferenceForTesting( |
| InputBufferKind pref) { |
| input_buffer_preference_ = pref; |
| } |
| |
| void VideoEncodeAcceleratorAdapter::Initialize(VideoCodecProfile profile, |
| const Options& options, |
| EncoderInfoCB info_cb, |
| OutputCB output_cb, |
| EncoderStatusCB done_cb) { |
| DCHECK(!accelerator_task_runner_->RunsTasksInCurrentSequence()); |
| accelerator_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce( |
| &VideoEncodeAcceleratorAdapter::InitializeOnAcceleratorThread, |
| base::Unretained(this), profile, options, |
| WrapCallback(std::move(info_cb)), WrapCallback(std::move(output_cb)), |
| WrapCallback(std::move(done_cb)))); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::InitializeOnAcceleratorThread( |
| VideoCodecProfile profile, |
| const Options& options, |
| EncoderInfoCB info_cb, |
| OutputCB output_cb, |
| EncoderStatusCB done_cb) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| if (state_ != State::kNotInitialized) { |
| std::move(done_cb).Run( |
| EncoderStatus(EncoderStatus::Codes::kEncoderInitializeTwice, |
| "Encoder has already been initialized.")); |
| return; |
| } |
| |
| accelerator_ = gpu_factories_->CreateVideoEncodeAccelerator(); |
| if (!accelerator_) { |
| std::move(done_cb).Run( |
| EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, |
| "Failed to create video encode accelerator.")); |
| return; |
| } |
| |
| if (options.frame_size.width() <= 0 || options.frame_size.height() <= 0) { |
| std::move(done_cb).Run( |
| EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig, |
| "Negative width or height values.")); |
| return; |
| } |
| |
| if (!options.frame_size.GetCheckedArea().IsValid()) { |
| std::move(done_cb).Run( |
| EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig, |
| "Frame is too large.")); |
| return; |
| } |
| |
| auto supported_profiles = |
| gpu_factories_->GetVideoEncodeAcceleratorSupportedProfiles(); |
| if (!supported_profiles) { |
| InitCompleted( |
| EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, |
| "No profile is supported by video encode accelerator.")); |
| return; |
| } |
| |
| auto supported_rc_modes = |
| VideoEncodeAccelerator::SupportedRateControlMode::kNoMode; |
| for (const auto& supported_profile : *supported_profiles) { |
| if (supported_profile.profile == profile) { |
| supported_rc_modes = supported_profile.rate_control_modes; |
| break; |
| } |
| } |
| |
| if (supported_rc_modes == |
| VideoEncodeAccelerator::SupportedRateControlMode::kNoMode) { |
| std::move(done_cb).Run(EncoderStatus( |
| EncoderStatus::Codes::kEncoderInitializationError, |
| "The profile is not supported by video encode accelerator.")); |
| return; |
| } |
| |
| profile_ = profile; |
| supported_rc_modes_ = supported_rc_modes; |
| options_ = options; |
| info_cb_ = std::move(info_cb); |
| output_cb_ = std::move(output_cb); |
| state_ = State::kWaitingForFirstFrame; |
| |
| #if BUILDFLAG(USE_PROPRIETARY_CODECS) |
| if (profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX && |
| !options_.avc.produce_annexb) { |
| h264_converter_ = std::make_unique<H264AnnexBToAvcBitstreamConverter>(); |
| } |
| #if BUILDFLAG(ENABLE_PLATFORM_HEVC) && \ |
| BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| if (profile_ == HEVCPROFILE_MAIN && !options_.hevc.produce_annexb) { |
| h265_converter_ = std::make_unique<H265AnnexBToHevcBitstreamConverter>(); |
| } |
| #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC) && |
| // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) |
| |
| std::move(done_cb).Run(EncoderStatus::Codes::kOk); |
| |
| // The accelerator will be initialized for real once we have the first frame. |
| } |
| |
| void VideoEncodeAcceleratorAdapter::InitializeInternalOnAcceleratorThread() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| DCHECK_EQ(state_, State::kWaitingForFirstFrame); |
| DCHECK(!pending_encodes_.empty()); |
| |
| // We use the first frame to setup the VEA config so that we can ensure that |
| // zero copy hardware encoding from the camera can be used. |
| const auto& first_frame = pending_encodes_.front()->frame; |
| const auto format = first_frame->format(); |
| const bool is_rgb = |
| format == PIXEL_FORMAT_XBGR || format == PIXEL_FORMAT_XRGB || |
| format == PIXEL_FORMAT_ABGR || format == PIXEL_FORMAT_ARGB; |
| const bool supported_format = |
| format == PIXEL_FORMAT_NV12 || format == PIXEL_FORMAT_I420 || is_rgb; |
| if (!supported_format) { |
| InitCompleted(EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode, |
| "Unexpected frame format.") |
| .WithData("frame", first_frame->AsHumanReadableString())); |
| return; |
| } |
| |
| auto vea_config = |
| SetUpVeaConfig(profile_, options_, format, first_frame->storage_type(), |
| supported_rc_modes_, required_encoder_type_); |
| |
| #if BUILDFLAG(IS_LINUX) || BUILDFLAG(IS_CHROMEOS) |
| // Linux/ChromeOS require a special configuration to use dmabuf storage. |
| // We need to keep sending frames the same way the first frame was sent. |
| // Other platforms will happily mix GpuMemoryBuffer storage with regular |
| // storage, so we don't care about mismatches on other platforms. |
| if (input_buffer_preference_ == InputBufferKind::Any) { |
| if (vea_config.storage_type == |
| VideoEncodeAccelerator::Config::StorageType::kGpuMemoryBuffer) { |
| input_buffer_preference_ = InputBufferKind::GpuMemBuf; |
| } else { |
| input_buffer_preference_ = InputBufferKind::CpuMemBuf; |
| } |
| } |
| #endif |
| if (!accelerator_->Initialize(vea_config, this, media_log_->Clone())) { |
| InitCompleted( |
| EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, |
| "Failed to initialize video encode accelerator.")); |
| return; |
| } |
| |
| state_ = State::kInitializing; |
| format_ = vea_config.input_format; |
| } |
| |
| void VideoEncodeAcceleratorAdapter::Encode(scoped_refptr<VideoFrame> frame, |
| const EncodeOptions& encode_options, |
| EncoderStatusCB done_cb) { |
| DCHECK(!accelerator_task_runner_->RunsTasksInCurrentSequence()); |
| accelerator_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce(&VideoEncodeAcceleratorAdapter::EncodeOnAcceleratorThread, |
| base::Unretained(this), std::move(frame), encode_options, |
| WrapCallback(std::move(done_cb)))); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::EncodeOnAcceleratorThread( |
| scoped_refptr<VideoFrame> frame, |
| EncodeOptions encode_options, |
| EncoderStatusCB done_cb) { |
| TRACE_EVENT1("media", |
| "VideoEncodeAcceleratorAdapter::EncodeOnAcceleratorThread", |
| "timestamp", frame->timestamp()); |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| |
| if (state_ == State::kWaitingForFirstFrame || |
| state_ == State::kInitializing) { |
| auto pending_encode = std::make_unique<PendingEncode>(); |
| pending_encode->done_callback = std::move(done_cb); |
| pending_encode->frame = std::move(frame); |
| pending_encode->options = encode_options; |
| pending_encodes_.push_back(std::move(pending_encode)); |
| if (state_ == State::kWaitingForFirstFrame) |
| InitializeInternalOnAcceleratorThread(); |
| return; |
| } |
| |
| if (state_ != State::kReadyToEncode) { |
| std::move(done_cb).Run( |
| EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode, |
| "Encoder can't encode now.")); |
| return; |
| } |
| |
| const bool frame_needs_resizing = |
| frame->visible_rect().size() != options_.frame_size || |
| frame->coded_size() != input_coded_size_; |
| |
| // Try using a frame with GPU buffer both are true: |
| // 1. the frame already has GPU buffer |
| // 2. frame doesn't need resizing or can be resized by GPU encoder. |
| bool use_gpu_buffer = frame->HasGpuMemoryBuffer() && |
| (!frame_needs_resizing || gpu_resize_supported_); |
| |
| // Currently configured encoder's preference takes precedence overe heuristic |
| // above. |
| if (input_buffer_preference_ == InputBufferKind::GpuMemBuf) |
| use_gpu_buffer = true; |
| if (input_buffer_preference_ == InputBufferKind::CpuMemBuf) |
| use_gpu_buffer = false; |
| |
| EncoderStatus::Or<scoped_refptr<VideoFrame>> result(nullptr); |
| if (use_gpu_buffer) |
| result = PrepareGpuFrame(frame); |
| else |
| result = PrepareCpuFrame(frame); |
| |
| if (!result.has_value()) { |
| std::move(done_cb).Run( |
| std::move(result) |
| .error() |
| .WithData("frame", frame->AsHumanReadableString()) |
| .AddHere()); |
| return; |
| } |
| |
| frame = std::move(result).value(); |
| |
| if (last_frame_color_space_ != frame->ColorSpace()) { |
| last_frame_color_space_ = frame->ColorSpace(); |
| encode_options.key_frame = true; |
| } |
| |
| auto active_encode = std::make_unique<PendingOp>(); |
| active_encode->done_callback = std::move(done_cb); |
| active_encode->timestamp = frame->timestamp(); |
| active_encode->color_space = frame->ColorSpace(); |
| active_encodes_.push_back(std::move(active_encode)); |
| accelerator_->Encode(frame, encode_options); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::ChangeOptions(const Options& options, |
| OutputCB output_cb, |
| EncoderStatusCB done_cb) { |
| DCHECK(!accelerator_task_runner_->RunsTasksInCurrentSequence()); |
| accelerator_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce( |
| &VideoEncodeAcceleratorAdapter::ChangeOptionsOnAcceleratorThread, |
| base::Unretained(this), options, WrapCallback(std::move(output_cb)), |
| WrapCallback(std::move(done_cb)))); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::ChangeOptionsOnAcceleratorThread( |
| const Options options, |
| OutputCB output_cb, |
| EncoderStatusCB done_cb) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| DCHECK(active_encodes_.empty()); |
| DCHECK(pending_encodes_.empty()); |
| CHECK(state_ == State::kReadyToEncode || |
| state_ == State::kWaitingForFirstFrame); |
| |
| if (options.bitrate && options_.bitrate && |
| options.bitrate->mode() != options_.bitrate->mode()) { |
| std::move(done_cb).Run( |
| EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, |
| "Bitrate mode change is not supported.")); |
| return; |
| } |
| |
| std::optional<gfx::Size> new_frame_size; |
| if (options.frame_size != options_.frame_size) { |
| if (supports_frame_size_change_) { |
| input_pool_.reset(); |
| gmb_frame_pool_.reset(); |
| new_frame_size = options.frame_size; |
| } else { |
| auto status = |
| EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, |
| "Resolution change is not supported."); |
| std::move(done_cb).Run(status); |
| return; |
| } |
| } |
| |
| Bitrate bitrate = |
| CreateBitrate(options.bitrate, options.frame_size, supported_rc_modes_); |
| uint32_t framerate = base::ClampRound<uint32_t>( |
| options.framerate.value_or(VideoEncodeAccelerator::kDefaultFramerate)); |
| |
| // When frame size is changed, run |done_cb| in |RequireBitstreamBuffers| |
| // after bitstream buffer is re-initialized. At that time, reconfigure is done |
| // and VEA is ready to accept new frames. |
| if (new_frame_size.has_value()) { |
| reconfigure_cb_ = std::move(done_cb); |
| } |
| state_ = State::kReconfiguring; |
| accelerator_->RequestEncodingParametersChange(bitrate, framerate, |
| new_frame_size); |
| |
| #if BUILDFLAG(USE_PROPRIETARY_CODECS) |
| if (profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX) { |
| if (options.avc.produce_annexb) { |
| h264_converter_.reset(); |
| } else if (!h264_converter_) { |
| h264_converter_ = std::make_unique<H264AnnexBToAvcBitstreamConverter>(); |
| } |
| } |
| #if BUILDFLAG(ENABLE_PLATFORM_HEVC) && \ |
| BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| if (profile_ == HEVCPROFILE_MAIN) { |
| if (options.hevc.produce_annexb) { |
| h265_converter_.reset(); |
| } else if (!h265_converter_) { |
| h265_converter_ = std::make_unique<H265AnnexBToHevcBitstreamConverter>(); |
| } |
| } |
| #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC) && |
| // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) |
| |
| options_ = options; |
| if (!output_cb.is_null()) |
| output_cb_ = std::move(output_cb); |
| if (!done_cb.is_null()) { |
| state_ = State::kReadyToEncode; |
| std::move(done_cb).Run(EncoderStatus::Codes::kOk); |
| } |
| } |
| |
| void VideoEncodeAcceleratorAdapter::Flush(EncoderStatusCB done_cb) { |
| DCHECK(!accelerator_task_runner_->RunsTasksInCurrentSequence()); |
| accelerator_task_runner_->PostTask( |
| FROM_HERE, |
| base::BindOnce(&VideoEncodeAcceleratorAdapter::FlushOnAcceleratorThread, |
| base::Unretained(this), WrapCallback(std::move(done_cb)))); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::FlushOnAcceleratorThread( |
| EncoderStatusCB done_cb) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| if (state_ == State::kWaitingForFirstFrame) { |
| // Nothing to do since we haven't actually initialized yet. |
| std::move(done_cb).Run(EncoderStatus::Codes::kOk); |
| return; |
| } |
| |
| if (state_ != State::kReadyToEncode && state_ != State::kInitializing) { |
| std::move(done_cb).Run(EncoderStatus( |
| EncoderStatus::Codes::kEncoderFailedFlush, "Encoder can't flush now")); |
| return; |
| } |
| |
| if (active_encodes_.empty() && pending_encodes_.empty()) { |
| // No active or pending encodes, nothing to flush. |
| std::move(done_cb).Run(EncoderStatus::Codes::kOk); |
| return; |
| } |
| |
| // When initializing the flush will be handled after pending encodes are sent. |
| if (state_ != State::kInitializing) { |
| DCHECK_EQ(state_, State::kReadyToEncode); |
| state_ = State::kFlushing; |
| } |
| |
| pending_flush_ = std::make_unique<PendingOp>(); |
| pending_flush_->done_callback = std::move(done_cb); |
| |
| // If flush is not supported FlushCompleted() will be called by |
| // BitstreamBufferReady() when |active_encodes_| is empty. |
| if (state_ == State::kFlushing && flush_support_.value()) { |
| accelerator_->Flush( |
| base::BindOnce(&VideoEncodeAcceleratorAdapter::FlushCompleted, |
| base::Unretained(this))); |
| } |
| } |
| |
| void VideoEncodeAcceleratorAdapter::RequireBitstreamBuffers( |
| unsigned int input_count, |
| const gfx::Size& input_coded_size, |
| size_t output_buffer_size) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| CHECK(state_ == State::kInitializing || state_ == State::kReconfiguring); |
| |
| input_coded_size_ = input_coded_size; |
| |
| constexpr int kOutputBufferNumber = 2; |
| output_buffer_handles_.clear(); |
| for (int id = 0; id < kOutputBufferNumber; id++) { |
| auto handle = output_pool_->MaybeAllocateBuffer(output_buffer_size); |
| if (!handle) { |
| if (state_ == State::kInitializing) { |
| InitCompleted(EncoderStatus::Codes::kEncoderInitializationError); |
| } else if (reconfigure_cb_) { |
| std::move(reconfigure_cb_) |
| .Run({EncoderStatus::Codes::kInvalidOutputBuffer, |
| "Failed to allocate buffer"}); |
| state_ = State::kNotInitialized; |
| } |
| return; |
| } |
| |
| const base::UnsafeSharedMemoryRegion& region = handle->GetRegion(); |
| accelerator_->UseOutputBitstreamBuffer( |
| BitstreamBuffer(id, region.Duplicate(), region.GetSize())); |
| output_buffer_handles_.push_back(std::move(handle)); |
| } |
| |
| if (state_ == State::kInitializing) { |
| InitCompleted(EncoderStatus::Codes::kOk); |
| } else if (state_ == State::kReconfiguring) { |
| state_ = State::kReadyToEncode; |
| if (reconfigure_cb_) { |
| std::move(reconfigure_cb_).Run(EncoderStatus::Codes::kOk); |
| } |
| } |
| } |
| |
| void VideoEncodeAcceleratorAdapter::BitstreamBufferReady( |
| int32_t buffer_id, |
| const BitstreamBufferMetadata& metadata) { |
| std::optional<CodecDescription> desc; |
| VideoEncoderOutput result; |
| result.key_frame = metadata.key_frame; |
| result.timestamp = metadata.timestamp; |
| result.size = metadata.payload_size_bytes; |
| if (metadata.h264.has_value()) |
| result.temporal_id = metadata.h264.value().temporal_idx; |
| else if (metadata.vp9.has_value()) |
| result.temporal_id = metadata.vp9.value().temporal_idx; |
| else if (metadata.vp8.has_value()) |
| result.temporal_id = metadata.vp8.value().temporal_idx; |
| else if (metadata.av1.has_value()) |
| result.temporal_id = metadata.av1.value().temporal_idx; |
| else if (metadata.h265.has_value()) |
| result.temporal_id = metadata.h265.value().temporal_idx; |
| |
| if (metadata.encoded_size) |
| result.encoded_size = metadata.encoded_size; |
| |
| if (buffer_id < 0 || |
| buffer_id >= static_cast<int>(output_buffer_handles_.size())) { |
| NotifyErrorStatus( |
| {EncoderStatus::Codes::kInvalidOutputBuffer, |
| "Buffer id is out of bounds: " + base::NumberToString(buffer_id)}); |
| } |
| if (!output_buffer_handles_[buffer_id]) { |
| NotifyErrorStatus( |
| {EncoderStatus::Codes::kInvalidOutputBuffer, "Invalid output buffer"}); |
| } |
| |
| const base::WritableSharedMemoryMapping& mapping = |
| output_buffer_handles_[buffer_id]->GetMapping(); |
| DCHECK_LE(result.size, mapping.size()); |
| |
| if (result.size > 0) { |
| bool stream_converted = false; |
| #if BUILDFLAG(USE_PROPRIETARY_CODECS) |
| uint8_t* src = static_cast<uint8_t*>(mapping.memory()); |
| size_t dst_size = result.size; |
| size_t actual_output_size = 0; |
| auto dst = std::make_unique<uint8_t[]>(dst_size); |
| bool config_changed = false; |
| media::MP4Status status; |
| if (h264_converter_) { |
| status = h264_converter_->ConvertChunk( |
| base::span<uint8_t>(src, result.size), |
| base::span<uint8_t>(dst.get(), dst_size), &config_changed, |
| &actual_output_size); |
| if (status.code() == MP4Status::Codes::kBufferTooSmall) { |
| // Between AnnexB and AVCC bitstream formats, the start code length and |
| // the nal size length can be different. See H.264 specification at |
| // http://www.itu.int/rec/T-REC-H.264. Retry the conversion if the |
| // output buffer size is too small. |
| dst_size = actual_output_size; |
| dst = std::make_unique<uint8_t[]>(dst_size); |
| status = h264_converter_->ConvertChunk( |
| base::span<uint8_t>(src, result.size), |
| base::span<uint8_t>(dst.get(), dst_size), &config_changed, |
| &actual_output_size); |
| } |
| |
| if (!status.is_ok()) { |
| NotifyErrorStatus({EncoderStatus::Codes::kEncoderFailedEncode, |
| "Failed to convert a buffer to h264 chunk"}); |
| return; |
| } |
| result.size = actual_output_size; |
| result.data = std::move(dst); |
| stream_converted = true; |
| |
| if (config_changed) { |
| const auto& config = h264_converter_->GetCurrentConfig(); |
| desc = CodecDescription(); |
| if (!config.Serialize(desc.value())) { |
| NotifyErrorStatus({media::EncoderStatus::Codes::kEncoderFailedEncode, |
| "Failed to get h264 config"}); |
| return; |
| } |
| } |
| } else { |
| #if BUILDFLAG(ENABLE_PLATFORM_HEVC) && \ |
| BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| if (h265_converter_) { |
| status = h265_converter_->ConvertChunk( |
| base::span<uint8_t>(src, result.size), |
| base::span<uint8_t>(dst.get(), dst_size), &config_changed, |
| &actual_output_size); |
| if (status.code() == MP4Status::Codes::kBufferTooSmall) { |
| dst_size = actual_output_size; |
| dst = std::make_unique<uint8_t[]>(dst_size); |
| status = h265_converter_->ConvertChunk( |
| base::span<uint8_t>(src, result.size), |
| base::span<uint8_t>(dst.get(), dst_size), &config_changed, |
| &actual_output_size); |
| } |
| |
| if (!status.is_ok()) { |
| NotifyErrorStatus({EncoderStatus::Codes::kEncoderFailedEncode, |
| "Failed to convert a buffer to h265 chunk"}); |
| return; |
| } |
| result.size = actual_output_size; |
| result.data = std::move(dst); |
| stream_converted = true; |
| |
| if (config_changed) { |
| const auto& config = h265_converter_->GetCurrentConfig(); |
| desc = CodecDescription(); |
| if (!config.Serialize(desc.value())) { |
| NotifyErrorStatus( |
| {media::EncoderStatus::Codes::kEncoderFailedEncode, |
| "Failed to get h265 config"}); |
| return; |
| } |
| } |
| } |
| #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC) && |
| // BUILDFLAG(ENABLE_HEVC_PARSER_AND_HW_DECODER) |
| } |
| #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) |
| if (!stream_converted) { |
| result.data = std::make_unique<uint8_t[]>(result.size); |
| memcpy(result.data.get(), mapping.memory(), result.size); |
| } |
| } |
| |
| // Give the buffer back to |accelerator_| |
| const base::UnsafeSharedMemoryRegion& region = |
| output_buffer_handles_[buffer_id]->GetRegion(); |
| accelerator_->UseOutputBitstreamBuffer( |
| BitstreamBuffer(buffer_id, region.Duplicate(), region.GetSize())); |
| |
| bool erased_active_encode = false; |
| for (auto it = active_encodes_.begin(); it != active_encodes_.end(); ++it) { |
| if ((*it)->timestamp == result.timestamp) { |
| result.color_space = |
| metadata.encoded_color_space.value_or((*it)->color_space); |
| std::move((*it)->done_callback).Run(EncoderStatus::Codes::kOk); |
| active_encodes_.erase(it); |
| erased_active_encode = true; |
| break; |
| } |
| } |
| DCHECK(erased_active_encode); |
| output_cb_.Run(std::move(result), std::move(desc)); |
| |
| if (active_encodes_.empty() && !flush_support_.value()) { |
| // Manually call FlushCompleted(), since |accelerator_| won't do it for us. |
| FlushCompleted(true); |
| } |
| } |
| |
| void VideoEncodeAcceleratorAdapter::NotifyErrorStatus( |
| const EncoderStatus& status) { |
| CHECK(!status.is_ok()); |
| MEDIA_LOG(ERROR, media_log_) |
| << "VEA adapter error. Code: " << static_cast<int32_t>(status.code()) |
| << ". Message: " << status.message(); |
| if (state_ == State::kInitializing) { |
| InitCompleted( |
| EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, |
| "VideoEncodeAccelerator encountered an error") |
| .WithData("VideoEncodeAccelerator status code", |
| static_cast<int32_t>(status.code()))); |
| return; |
| } |
| |
| if (state_ == State::kReconfiguring) { |
| if (reconfigure_cb_) { |
| std::move(reconfigure_cb_) |
| .Run(EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, |
| "VideoEncodeAccelerator encountered an error") |
| .WithData("VideoEncodeAccelerator status code", |
| static_cast<int32_t>(status.code()))); |
| } |
| state_ = State::kNotInitialized; |
| return; |
| } |
| |
| if (state_ == State::kFlushing) |
| FlushCompleted(false); |
| |
| // Report the error to all encoding-done callbacks |
| for (auto& encode : active_encodes_) { |
| std::move(encode->done_callback) |
| .Run(EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode, |
| "VideoEncodeAccelerator encountered an error") |
| .WithData("VideoEncodeAccelerator status code", |
| static_cast<int32_t>(status.code()))); |
| } |
| active_encodes_.clear(); |
| state_ = State::kNotInitialized; |
| } |
| |
| void VideoEncodeAcceleratorAdapter::NotifyEncoderInfoChange( |
| const VideoEncoderInfo& info) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| supports_frame_size_change_ = info.supports_frame_size_change; |
| info_cb_.Run(info); |
| } |
| |
| void VideoEncodeAcceleratorAdapter::InitCompleted(EncoderStatus status) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| |
| if (!status.is_ok()) { |
| // Report the error to all encoding-done callbacks |
| for (auto& encode : pending_encodes_) |
| std::move(encode->done_callback).Run(status); |
| |
| if (pending_flush_) |
| FlushCompleted(false); |
| |
| DCHECK(active_encodes_.empty()); |
| pending_encodes_.clear(); |
| state_ = State::kNotInitialized; |
| return; |
| } |
| |
| state_ = State::kReadyToEncode; |
| flush_support_ = accelerator_->IsFlushSupported(); |
| gpu_resize_supported_ = accelerator_->IsGpuFrameResizeSupported(); |
| |
| // Send off the encodes that came in while we were waiting for initialization. |
| for (auto& encode : pending_encodes_) { |
| EncodeOnAcceleratorThread(std::move(encode->frame), encode->options, |
| std::move(encode->done_callback)); |
| } |
| pending_encodes_.clear(); |
| |
| // If a Flush() came in during initialization, transition to flushing now that |
| // all the pending encodes have been sent. |
| if (pending_flush_) { |
| state_ = State::kFlushing; |
| if (flush_support_.value()) { |
| accelerator_->Flush( |
| base::BindOnce(&VideoEncodeAcceleratorAdapter::FlushCompleted, |
| base::Unretained(this))); |
| } |
| } |
| } |
| |
| void VideoEncodeAcceleratorAdapter::FlushCompleted(bool success) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| if (!pending_flush_) |
| return; |
| |
| std::move(pending_flush_->done_callback) |
| .Run(success ? EncoderStatus::Codes::kOk |
| : EncoderStatus::Codes::kEncoderFailedFlush); |
| pending_flush_.reset(); |
| state_ = State::kReadyToEncode; |
| } |
| |
| template <class T> |
| T VideoEncodeAcceleratorAdapter::WrapCallback(T cb) { |
| DCHECK(callback_task_runner_); |
| if (cb.is_null()) |
| return cb; |
| return base::BindPostTask(callback_task_runner_, std::move(cb)); |
| } |
| |
| // Copy a frame into a shared mem buffer and resize it as the same time. Input |
| // frames can I420, NV12, or RGB -- they'll be converted to I420 if needed. |
| EncoderStatus::Or<scoped_refptr<VideoFrame>> |
| VideoEncodeAcceleratorAdapter::PrepareCpuFrame( |
| scoped_refptr<VideoFrame> src_frame) { |
| TRACE_EVENT0("media", "VideoEncodeAcceleratorAdapter::PrepareCpuFrame"); |
| |
| const auto dest_coded_size = input_coded_size_; |
| const auto dest_visible_rect = gfx::Rect(options_.frame_size); |
| |
| // The frame whose storage type is STORAGE_OWNED_MEMORY and |
| // STORAGE_UNOWNED_MEMORY is copied here, not in mojo_video_frame_traits. |
| // It is because VEAAdapter recycles the SharedMemoryRegion, but |
| // mojo_video_frame_traits doesn't. |
| if (src_frame->storage_type() == VideoFrame::STORAGE_SHMEM && |
| src_frame->format() == PIXEL_FORMAT_I420 && |
| src_frame->visible_rect() == dest_visible_rect && |
| src_frame->coded_size() == dest_coded_size) { |
| // Nothing to do here, the input frame is already what we need. |
| return src_frame; |
| } |
| |
| if (!input_pool_) { |
| const size_t input_buffer_size = |
| VideoFrame::AllocationSize(PIXEL_FORMAT_I420, dest_coded_size); |
| input_pool_ = base::MakeRefCounted<ReadOnlyRegionPool>(input_buffer_size); |
| } |
| |
| std::unique_ptr<ReadOnlyRegionPool::Handle> handle = |
| input_pool_->MaybeAllocateBuffer(); |
| if (!handle || !handle->IsValid()) |
| return EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode); |
| |
| const base::WritableSharedMemoryMapping* mapping = handle->mapping(); |
| auto mapped_src_frame = src_frame->HasGpuMemoryBuffer() |
| ? ConvertToMemoryMappedFrame(src_frame) |
| : src_frame; |
| auto shared_frame = VideoFrame::WrapExternalData( |
| PIXEL_FORMAT_I420, dest_coded_size, dest_visible_rect, |
| dest_visible_rect.size(), static_cast<uint8_t*>(mapping->memory()), |
| mapping->size(), src_frame->timestamp()); |
| |
| if (!shared_frame || !mapped_src_frame) |
| return EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode); |
| |
| auto status = |
| frame_converter_.ConvertAndScale(*mapped_src_frame, *shared_frame); |
| if (!status.is_ok()) { |
| return EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode) |
| .AddCause(std::move(status)); |
| } |
| |
| shared_frame->BackWithSharedMemory(handle->region()); |
| shared_frame->AddDestructionObserver( |
| base::DoNothingWithBoundArgs(std::move(handle))); |
| return shared_frame; |
| } |
| |
| // Copy a frame into a GPU buffer and resize it as the same time. Input frames |
| // can I420, NV12, or RGB -- they'll be converted to NV12 if needed. |
| EncoderStatus::Or<scoped_refptr<VideoFrame>> |
| VideoEncodeAcceleratorAdapter::PrepareGpuFrame( |
| scoped_refptr<VideoFrame> src_frame) { |
| TRACE_EVENT0("media", "VideoEncodeAcceleratorAdapter::PrepareGpuFrame"); |
| DCHECK_CALLED_ON_VALID_SEQUENCE(accelerator_sequence_checker_); |
| DCHECK(src_frame); |
| |
| const auto dest_coded_size = input_coded_size_; |
| const auto dest_visible_rect = gfx::Rect(options_.frame_size); |
| |
| if (src_frame->HasGpuMemoryBuffer() && |
| src_frame->format() == PIXEL_FORMAT_NV12 && |
| (gpu_resize_supported_ || src_frame->coded_size() == dest_coded_size)) { |
| // Nothing to do here, the input frame is already what we need |
| return src_frame; |
| } |
| |
| if (!gmb_frame_pool_) { |
| gmb_frame_pool_ = base::MakeRefCounted<GpuMemoryBufferVideoFramePool>( |
| gpu_factories_, dest_coded_size); |
| } |
| |
| auto gpu_frame = |
| gmb_frame_pool_->MaybeCreateVideoFrame(dest_visible_rect.size()); |
| if (!gpu_frame) |
| return EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode); |
| |
| gpu_frame->set_timestamp(src_frame->timestamp()); |
| gpu_frame->metadata().MergeMetadataFrom(src_frame->metadata()); |
| |
| // Don't be scared. ConvertToMemoryMappedFrame() doesn't copy pixel data |
| // it just maps GPU buffer owned by |gpu_frame| and presents it as mapped |
| // view in CPU memory. It allows us to use ConvertAndScale() without |
| // having to tinker with libyuv and GpuMemoryBuffer memory views. |
| // |mapped_gpu_frame| doesn't own anything, but unmaps the buffer when freed. |
| // This is true because |gpu_frame| is created with |
| // |VEA_READ_CAMERA_AND_CPU_READ_WRITE| usage flag. |
| auto mapped_gpu_frame = ConvertToMemoryMappedFrame(gpu_frame); |
| auto mapped_src_frame = src_frame->HasGpuMemoryBuffer() |
| ? ConvertToMemoryMappedFrame(src_frame) |
| : src_frame; |
| if (!mapped_gpu_frame || !mapped_src_frame) |
| return EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode); |
| |
| auto status = |
| frame_converter_.ConvertAndScale(*mapped_src_frame, *mapped_gpu_frame); |
| if (!status.is_ok()) { |
| return EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode) |
| .AddCause(std::move(status)); |
| } |
| |
| // |mapped_gpu_frame| has the color space respecting the color conversion in |
| // ConvertAndScale(). |
| #if BUILDFLAG(IS_MAC) |
| gpu_frame->GetGpuMemoryBuffer()->SetColorSpace( |
| mapped_gpu_frame->ColorSpace()); |
| #endif |
| gpu_frame->set_color_space(mapped_gpu_frame->ColorSpace()); |
| |
| return gpu_frame; |
| } |
| |
| } // namespace media |