Reset current frame of video provider when context is lost

When user switch to another applicaiton during user is using
browser, system LMK can kill browser's gpu process if other app
require memory. This is natural situation in Android.

When user goes to HOME, if there was html video in browser's
page, video frame's resources are lost when gpu proces is killed.

When user return to browser, gpu proces is recreated and context
is also recreated. VideoFrameSubmitter start submitting when it
receive the new context. But VideoFrameProvider still has current
frame which gpu resources are already lost, so VideoFrameSubmitter
submitt invalid frame. This invalid frame cause currption on the
screen.

This patch reset the current frame of VideoFrameProvider when
context is lost, so that VideoFrameSumitter will not submit invalid
frame.

Bug: 1324547
Change-Id: I83e302e16d25fac2f3e6b4a252f13d7b0bce16e3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3642076
Reviewed-by: Stephen McGruer <smcgruer@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Henrik Boström <hbos@chromium.org>
Commit-Queue: Sangheon Kim <sangheon77.kim@samsung.com>
Reviewed-by: Will Cassella <cassew@chromium.org>
Cr-Commit-Position: refs/heads/main@{#1006860}

diff --git a/cc/layers/video_frame_provider.h b/cc/layers/video_frame_provider.h
index 7ae0890..7dff50c3 100644
--- a/cc/layers/video_frame_provider.h
+++ b/cc/layers/video_frame_provider.h

@@ -97,6 +97,10 @@
   // the client.
   virtual base::TimeDelta GetPreferredRenderInterval() = 0;
 
+  // Inform the provider that the context is lost. The provider need to reset
+  // the current frame if it's invald.
+  virtual void OnContextLost() = 0;
+
  protected:
   virtual ~VideoFrameProvider() {}
 };

diff --git a/cc/test/fake_video_frame_provider.h b/cc/test/fake_video_frame_provider.h
index 5fe8354d..58ed7cf 100644
--- a/cc/test/fake_video_frame_provider.h
+++ b/cc/test/fake_video_frame_provider.h

@@ -24,6 +24,7 @@
   scoped_refptr<media::VideoFrame> GetCurrentFrame() override;
   void PutCurrentFrame() override;
   base::TimeDelta GetPreferredRenderInterval() override;
+  void OnContextLost() override {}
 
   Client* client() { return client_; }
 

diff --git a/third_party/blink/public/platform/media/video_frame_compositor.h b/third_party/blink/public/platform/media/video_frame_compositor.h
index 391ea0a..13f8b5e 100644
--- a/third_party/blink/public/platform/media/video_frame_compositor.h
+++ b/third_party/blink/public/platform/media/video_frame_compositor.h

@@ -109,6 +109,7 @@
   scoped_refptr<media::VideoFrame> GetCurrentFrame() override;
   void PutCurrentFrame() override;
   base::TimeDelta GetPreferredRenderInterval() override;
+  void OnContextLost() override;
 
   // Returns |current_frame_|, without offering a guarantee as to how recently
   // it was updated. In certain applications, one might need to periodically

diff --git a/third_party/blink/renderer/modules/mediastream/DEPS b/third_party/blink/renderer/modules/mediastream/DEPS
index 2476259..35cbf47 100644
--- a/third_party/blink/renderer/modules/mediastream/DEPS
+++ b/third_party/blink/renderer/modules/mediastream/DEPS

@@ -59,6 +59,7 @@
         "+base/containers/circular_deque.h",
         "+base/run_loop.h",
         "+cc/layers/layer.h",
+        "+media/video/fake_gpu_memory_buffer.h",
         "+media/video/mock_gpu_memory_buffer_video_frame_pool.h",
         "+media/video/mock_gpu_video_accelerator_factories.h",
 

diff --git a/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.cc b/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.cc
index ab6e4e3..efe69f0 100644
--- a/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.cc
+++ b/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.cc

@@ -583,6 +583,21 @@
   return rendering_frame_buffer_->average_frame_duration();
 }
 
+void WebMediaPlayerMSCompositor::OnContextLost() {
+  DCHECK(video_frame_compositor_task_runner_->BelongsToCurrentThread());
+  // current_frame_'s resource in the context has been lost, so current_frame_
+  // is not valid any more. current_frame_ should be reset. Now the compositor
+  // has no concept of resetting current_frame_, so a black frame is set.
+  base::AutoLock auto_lock(current_frame_lock_);
+  if (!current_frame_ || (!current_frame_->HasTextures() &&
+                          !current_frame_->HasGpuMemoryBuffer())) {
+    return;
+  }
+  scoped_refptr<media::VideoFrame> black_frame =
+      media::VideoFrame::CreateBlackFrame(current_frame_->natural_size());
+  SetCurrentFrame(std::move(black_frame), false, absl::nullopt);
+}
+
 void WebMediaPlayerMSCompositor::StartRendering() {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
   {

diff --git a/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.h b/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.h
index d4facc1..603f9c5 100644
--- a/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.h
+++ b/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_compositor.h

@@ -113,6 +113,7 @@
   scoped_refptr<media::VideoFrame> GetCurrentFrame() override;
   void PutCurrentFrame() override;
   base::TimeDelta GetPreferredRenderInterval() override;
+  void OnContextLost() override;
 
   void StartRendering();
   void StopRendering();

diff --git a/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_test.cc b/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_test.cc
index bc9eecc..0b7231d5 100644
--- a/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_test.cc
+++ b/third_party/blink/renderer/modules/mediastream/webmediaplayer_ms_test.cc

@@ -20,6 +20,7 @@
 #include "media/base/media_util.h"
 #include "media/base/test_helpers.h"
 #include "media/base/video_frame.h"
+#include "media/video/fake_gpu_memory_buffer.h"
 #include "media/video/mock_gpu_memory_buffer_video_frame_pool.h"
 #include "media/video/mock_gpu_video_accelerator_factories.h"
 #include "third_party/blink/public/common/media/display_type.h"
@@ -1446,6 +1447,34 @@
   EXPECT_GE(compositor_->GetPreferredRenderInterval(), base::TimeDelta());
 }
 
+TEST_P(WebMediaPlayerMSTest, OnContextLost) {
+  InitializeWebMediaPlayerMS();
+  LoadAndGetFrameProvider(true);
+
+  gfx::Size frame_size(320, 240);
+  auto non_gpu_frame = media::VideoFrame::CreateZeroInitializedFrame(
+      media::PIXEL_FORMAT_I420, frame_size, gfx::Rect(frame_size), frame_size,
+      base::Seconds(10));
+  compositor_->EnqueueFrame(non_gpu_frame, true);
+  base::RunLoop().RunUntilIdle();
+  // frame without gpu resource should be remained even though context is lost
+  compositor_->OnContextLost();
+  EXPECT_EQ(non_gpu_frame, compositor_->GetCurrentFrame());
+
+  std::unique_ptr<gfx::GpuMemoryBuffer> gmb =
+      std::make_unique<media::FakeGpuMemoryBuffer>(
+          frame_size, gfx::BufferFormat::YUV_420_BIPLANAR);
+  gpu::MailboxHolder mailbox_holders[media::VideoFrame::kMaxPlanes];
+  auto gpu_frame = media::VideoFrame::WrapExternalGpuMemoryBuffer(
+      gfx::Rect(frame_size), frame_size, std::move(gmb), mailbox_holders,
+      base::DoNothing(), base::TimeDelta());
+  compositor_->EnqueueFrame(gpu_frame, true);
+  base::RunLoop().RunUntilIdle();
+  // frame with gpu resource should be reset if context is lost
+  compositor_->OnContextLost();
+  EXPECT_NE(gpu_frame, compositor_->GetCurrentFrame());
+}
+
 INSTANTIATE_TEST_SUITE_P(All,
                          WebMediaPlayerMSTest,
                          ::testing::Combine(::testing::Bool(),

diff --git a/third_party/blink/renderer/platform/graphics/video_frame_submitter.cc b/third_party/blink/renderer/platform/graphics/video_frame_submitter.cc
index e7001a1..c6ea218 100644
--- a/third_party/blink/renderer/platform/graphics/video_frame_submitter.cc
+++ b/third_party/blink/renderer/platform/graphics/video_frame_submitter.cc

@@ -290,6 +290,9 @@
   waiting_for_compositor_ack_ = false;
   last_frame_id_.reset();
 
+  if (video_frame_provider_)
+    video_frame_provider_->OnContextLost();
+
   resource_provider_->OnContextLost();
 
   // NOTE: These objects should be reset last; and if `bundle_proxy`_ is set, it
@@ -526,7 +529,6 @@
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);
   if (!compositor_frame_sink_)
     return;
-
   const auto is_driving_frame_updates = IsDrivingFrameUpdates();
   compositor_frame_sink_->SetNeedsBeginFrame(is_driving_frame_updates);
   power_mode_voter_->VoteFor(power_scheduler::PowerMode::kVideoPlayback);

diff --git a/third_party/blink/renderer/platform/graphics/video_frame_submitter_test.cc b/third_party/blink/renderer/platform/graphics/video_frame_submitter_test.cc
index 437691d..b6f42bc 100644
--- a/third_party/blink/renderer/platform/graphics/video_frame_submitter_test.cc
+++ b/third_party/blink/renderer/platform/graphics/video_frame_submitter_test.cc

@@ -59,6 +59,7 @@
   MOCK_METHOD0(HasCurrentFrame, bool());
   MOCK_METHOD0(GetCurrentFrame, scoped_refptr<media::VideoFrame>());
   MOCK_METHOD0(PutCurrentFrame, void());
+  MOCK_METHOD0(OnContextLost, void());
 
   base::TimeDelta GetPreferredRenderInterval() override {
     return preferred_interval;
@@ -698,6 +699,7 @@
       .Times(0);
   EXPECT_CALL(mock_embedded_frame_sink_provider, CreateCompositorFrameSink_(_))
       .Times(1);
+  EXPECT_CALL(*video_frame_provider_, OnContextLost()).Times(1);
   submitter_->OnContextLost();
   OnReceivedContextProvider(true, context_provider_);
   task_environment_.RunUntilIdle();
@@ -719,6 +721,7 @@
       .Times(0);
   EXPECT_CALL(mock_embedded_frame_sink_provider, CreateCompositorFrameSink_(_))
       .Times(1);
+  EXPECT_CALL(*video_frame_provider_, OnContextLost()).Times(1);
   submitter_->OnContextLost();
   OnReceivedContextProvider(false, nullptr);
   task_environment_.RunUntilIdle();

diff --git a/third_party/blink/renderer/platform/media/DEPS b/third_party/blink/renderer/platform/media/DEPS
index cee97c54..4401bda 100644
--- a/third_party/blink/renderer/platform/media/DEPS
+++ b/third_party/blink/renderer/platform/media/DEPS

@@ -46,6 +46,7 @@
     "+components/viz/test",
     "+media/mojo/services",
     "+media/renderers",
+    "+media/video/fake_gpu_memory_buffer.h",
     "+gin/v8_initializer.h",
     "+mojo/core/embedder/embedder.h",
 

diff --git a/third_party/blink/renderer/platform/media/video_frame_compositor.cc b/third_party/blink/renderer/platform/media/video_frame_compositor.cc
index 2e30bda..4cfa47f 100644
--- a/third_party/blink/renderer/platform/media/video_frame_compositor.cc
+++ b/third_party/blink/renderer/platform/media/video_frame_compositor.cc

@@ -462,4 +462,19 @@
   return new_frame || had_new_background_frame;
 }
 
+void VideoFrameCompositor::OnContextLost() {
+  DCHECK(task_runner_->BelongsToCurrentThread());
+  // current_frame_'s resource in the context has been lost, so current_frame_
+  // is not valid any more. current_frame_ should be reset. Now the compositor
+  // has no concept of resetting current_frame_, so a black frame is set.
+  base::AutoLock lock(current_frame_lock_);
+  if (!current_frame_ || (!current_frame_->HasTextures() &&
+                          !current_frame_->HasGpuMemoryBuffer())) {
+    return;
+  }
+  scoped_refptr<media::VideoFrame> black_frame =
+      media::VideoFrame::CreateBlackFrame(current_frame_->natural_size());
+  SetCurrentFrame_Locked(std::move(black_frame), tick_clock_->NowTicks());
+}
+
 }  // namespace blink

diff --git a/third_party/blink/renderer/platform/media/video_frame_compositor_unittest.cc b/third_party/blink/renderer/platform/media/video_frame_compositor_unittest.cc
index 81f02b3..5893c7cd 100644
--- a/third_party/blink/renderer/platform/media/video_frame_compositor_unittest.cc
+++ b/third_party/blink/renderer/platform/media/video_frame_compositor_unittest.cc

@@ -16,6 +16,7 @@
 #include "components/viz/common/frame_sinks/begin_frame_args.h"
 #include "components/viz/common/surfaces/frame_sink_id.h"
 #include "media/base/video_frame.h"
+#include "media/video/fake_gpu_memory_buffer.h"
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/blink/public/platform/web_video_frame_submitter.h"
@@ -492,4 +493,46 @@
             viz::BeginFrameArgs::MinInterval());
 }
 
+TEST_F(VideoFrameCompositorTest, OnContextLost) {
+  scoped_refptr<media::VideoFrame> non_gpu_frame = CreateOpaqueFrame();
+
+  gfx::Size encode_size(320, 240);
+  std::unique_ptr<gfx::GpuMemoryBuffer> gmb =
+      std::make_unique<media::FakeGpuMemoryBuffer>(
+          encode_size, gfx::BufferFormat::YUV_420_BIPLANAR);
+  gpu::MailboxHolder mailbox_holders[media::VideoFrame::kMaxPlanes];
+  scoped_refptr<media::VideoFrame> gpu_frame =
+      media::VideoFrame::WrapExternalGpuMemoryBuffer(
+          gfx::Rect(encode_size), encode_size, std::move(gmb), mailbox_holders,
+          base::DoNothing(), base::TimeDelta());
+
+  compositor_->set_background_rendering_for_testing(true);
+
+  EXPECT_CALL(*submitter_, IsDrivingFrameUpdates)
+      .Times(AnyNumber())
+      .WillRepeatedly(Return(true));
+
+  // Move the clock forward. Otherwise, the current time will be 0, will appear
+  // null, and will cause DCHECKs.
+  tick_clock_.Advance(base::Seconds(1));
+
+  EXPECT_CALL(*this, Render(_, _, RenderingMode::kStartup))
+      .WillOnce(Return(non_gpu_frame));
+  StartVideoRendererSink();
+  compositor()->OnContextLost();
+  // frame which dose not have gpu resource should be maintained even though
+  // context is lost.
+  EXPECT_EQ(non_gpu_frame, compositor()->GetCurrentFrame());
+
+  tick_clock_.Advance(base::Seconds(1));
+  EXPECT_CALL(*this, Render(_, _, _)).WillOnce(Return(gpu_frame));
+  compositor()->UpdateCurrentFrameIfStale(
+      VideoFrameCompositor::UpdateType::kBypassClient);
+  compositor()->OnContextLost();
+  // frame which has gpu resource should be reset if context is lost
+  EXPECT_NE(gpu_frame, compositor()->GetCurrentFrame());
+
+  StopVideoRendererSink(true);
+}
+
 }  // namespace blink