[go: nahoru, domu]
Google Git
Sign in
chromium / chromium / src / 33fd71f0cc028eebdc456b115c2cc23a6c31514b / . / media / renderers / renderer_impl.h
blob: 8dd6ffe910f3ed0e915efb27cf5cbae8c26d18e0 [file] [log] [blame]
Avi Drissmand387f0922022-09-14 20:51:31[diff] [blame]1// Copyright 2014 The Chromium Authors
xhwangbe9da702014-08-23 21:44:55[diff] [blame]2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
servolkf54f5c8f2015-02-24 20:32:39[diff] [blame]5#ifndef MEDIA_RENDERERS_RENDERER_IMPL_H_
6#define MEDIA_RENDERERS_RENDERER_IMPL_H_
xhwangbe9da702014-08-23 21:44:55[diff] [blame]7
danakj4d43bc22016-04-26 03:36:04[diff] [blame]8#include <memory>
dalecurtise92934582015-05-12 06:38:00[diff] [blame]9#include <vector>
10
dalecurtisc11e7bb2015-04-03 05:07:08[diff] [blame]11#include "base/cancelable_callback.h"
Keishi Hattori0e45c022021-11-27 09:25:52[diff] [blame]12#include "base/memory/raw_ptr.h"
Lei Zhang9f7e05d162022-11-07 18:43:37[diff] [blame]13#include "base/memory/scoped_refptr.h"
xhwangbe9da702014-08-23 21:44:55[diff] [blame]14#include "base/memory/weak_ptr.h"
15#include "base/synchronization/lock.h"
Markus Handell2878995f2022-10-14 07:32:09[diff] [blame]16#include "base/task/sequenced_task_runner.h"
xhwangbe9da702014-08-23 21:44:55[diff] [blame]17#include "base/time/clock.h"
18#include "base/time/default_tick_clock.h"
19#include "base/time/time.h"
Chris Cunningham038548b2017-07-10 22:36:30[diff] [blame]20#include "media/base/audio_decoder_config.h"
xhwangbe9da702014-08-23 21:44:55[diff] [blame]21#include "media/base/buffering_state.h"
xhwang97de4202014-11-25 08:44:01[diff] [blame]22#include "media/base/decryptor.h"
alokp16bbeea2016-05-12 23:32:36[diff] [blame]23#include "media/base/demuxer_stream.h"
xhwangbe9da702014-08-23 21:44:55[diff] [blame]24#include "media/base/media_export.h"
25#include "media/base/pipeline_status.h"
26#include "media/base/renderer.h"
liberato@chromium.org8705ffc2020-07-28 20:11:55[diff] [blame]27#include "media/base/tuneable.h"
Chris Cunningham038548b2017-07-10 22:36:30[diff] [blame]28#include "media/base/video_decoder_config.h"
Xiaohan Wang640b41d2018-12-18 19:00:46[diff] [blame]29#include "media/base/waiting.h"
alokp5d86e9b2016-05-17 20:20:41[diff] [blame]30#include "ui/gfx/geometry/size.h"
xhwangbe9da702014-08-23 21:44:55[diff] [blame]31
xhwangbe9da702014-08-23 21:44:55[diff] [blame]32namespace media {
33
34class AudioRenderer;
servolkcfc91f152017-02-02 05:11:23[diff] [blame]35class MediaResource;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]36class TimeSource;
37class VideoRenderer;
scherkusece89452014-09-09 23:13:38[diff] [blame]38class WallClockTimeSource;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]39
Zentaro Kavanagh19d10592020-11-03 23:26:14[diff] [blame]40class MEDIA_EXPORT RendererImpl final : public Renderer {
xhwangbe9da702014-08-23 21:44:55[diff] [blame]41 public:
xhwangabd95fd2014-10-03 07:10:01[diff] [blame]42 // Renders audio/video streams using |audio_renderer| and |video_renderer|
43 // provided. All methods except for GetMediaTime() run on the |task_runner|.
44 // GetMediaTime() runs on the render main thread because it's part of JS sync
45 // API.
Markus Handell2878995f2022-10-14 07:32:09[diff] [blame]46 RendererImpl(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
danakj4d43bc22016-04-26 03:36:04[diff] [blame]47 std::unique_ptr<AudioRenderer> audio_renderer,
48 std::unique_ptr<VideoRenderer> video_renderer);
xhwangbe9da702014-08-23 21:44:55[diff] [blame]49
Peter Boström4c42559f2021-09-22 20:24:34[diff] [blame]50 RendererImpl(const RendererImpl&) = delete;
51 RendererImpl& operator=(const RendererImpl&) = delete;
52
xhwang97de4202014-11-25 08:44:01[diff] [blame]53 ~RendererImpl() final;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]54
55 // Renderer implementation.
servolkcfc91f152017-02-02 05:11:23[diff] [blame]56 void Initialize(MediaResource* media_resource,
alokp16bbeea2016-05-12 23:32:36[diff] [blame]57 RendererClient* client,
Chris Mumford84f095f2019-10-25 23:19:44[diff] [blame]58 PipelineStatusCallback init_cb) final;
Takashi Sakamotocfb66262019-10-30 09:19:09[diff] [blame]59 void SetCdm(CdmContext* cdm_context, CdmAttachedCB cdm_attached_cb) final;
Arthur Sonzogni581b9e82024-01-30 17:25:15[diff] [blame]60 void SetLatencyHint(std::optional<base::TimeDelta> latency_hint) final;
Thomas Guilbertdd0e3e62020-06-24 23:53:42[diff] [blame]61 void SetPreservesPitch(bool preserves_pitch) final;
evliu4a95d632022-01-04 20:54:51[diff] [blame]62 void SetWasPlayedWithUserActivation(
63 bool was_played_with_user_activation) final;
Chris Mumford1ccf4512019-10-25 19:34:05[diff] [blame]64 void Flush(base::OnceClosure flush_cb) final;
xhwang97de4202014-11-25 08:44:01[diff] [blame]65 void StartPlayingFrom(base::TimeDelta time) final;
a.berwal338bf002015-04-22 11:14:50[diff] [blame]66 void SetPlaybackRate(double playback_rate) final;
xhwang97de4202014-11-25 08:44:01[diff] [blame]67 void SetVolume(float volume) final;
68 base::TimeDelta GetMediaTime() final;
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]69 void OnSelectedVideoTracksChanged(
70 const std::vector<DemuxerStream*>& enabled_tracks,
Xiaohan Wang092b4e9a2022-11-02 18:15:56[diff] [blame]71 base::OnceClosure change_completed_cb) final;
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]72 void OnEnabledAudioTracksChanged(
73 const std::vector<DemuxerStream*>& enabled_tracks,
Xiaohan Wang092b4e9a2022-11-02 18:15:56[diff] [blame]74 base::OnceClosure change_completed_cb) final;
75 RendererType GetRendererType() final;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]76
77 // Helper functions for testing purposes. Must be called before Initialize().
78 void DisableUnderflowForTesting();
scherkusece89452014-09-09 23:13:38[diff] [blame]79 void EnableClocklessVideoPlaybackForTesting();
dalecurtisc11e7bb2015-04-03 05:07:08[diff] [blame]80 void set_time_source_for_testing(TimeSource* time_source) {
81 time_source_ = time_source;
82 }
83 void set_video_underflow_threshold_for_testing(base::TimeDelta threshold) {
liberato@chromium.org8705ffc2020-07-28 20:11:55[diff] [blame]84 video_underflow_threshold_.set_for_testing(threshold);
dalecurtisc11e7bb2015-04-03 05:07:08[diff] [blame]85 }
xhwangbe9da702014-08-23 21:44:55[diff] [blame]86
87 private:
alokp16bbeea2016-05-12 23:32:36[diff] [blame]88 class RendererClientInternal;
89
xhwangbe9da702014-08-23 21:44:55[diff] [blame]90 enum State {
91 STATE_UNINITIALIZED,
xhwanga935e442016-02-11 02:22:45[diff] [blame]92 STATE_INIT_PENDING_CDM, // Initialization is waiting for the CDM to be set.
93 STATE_INITIALIZING, // Initializing audio/video renderers.
servolka1597ec2017-04-06 19:49:19[diff] [blame]94 STATE_FLUSHING, // Flushing is in progress.
95 STATE_FLUSHED, // After initialization or after flush completed.
96 STATE_PLAYING, // After StartPlayingFrom has been called.
xhwangbe9da702014-08-23 21:44:55[diff] [blame]97 STATE_ERROR
98 };
99
dalecurtise92934582015-05-12 06:38:00[diff] [blame]100 bool GetWallClockTimes(const std::vector<base::TimeDelta>& media_timestamps,
101 std::vector<base::TimeTicks>* wall_clock_times);
xhwangbe9da702014-08-23 21:44:55[diff] [blame]102
xhwanga935e442016-02-11 02:22:45[diff] [blame]103 bool HasEncryptedStream();
104
105 void FinishInitialization(PipelineStatus status);
Dale Curtise130ef92018-09-20 00:17:13[diff] [blame]106 void FinishFlush();
xhwang97de4202014-11-25 08:44:01[diff] [blame]107
xhwangbe9da702014-08-23 21:44:55[diff] [blame]108 // Helper functions and callbacks for Initialize().
109 void InitializeAudioRenderer();
110 void OnAudioRendererInitializeDone(PipelineStatus status);
111 void InitializeVideoRenderer();
112 void OnVideoRendererInitializeDone(PipelineStatus status);
113
114 // Helper functions and callbacks for Flush().
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]115 void FlushInternal();
xhwangbe9da702014-08-23 21:44:55[diff] [blame]116 void FlushAudioRenderer();
117 void OnAudioRendererFlushDone();
118 void FlushVideoRenderer();
119 void OnVideoRendererFlushDone();
120
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]121 // Reinitialize audio/video renderer during a demuxer stream switching. The
122 // renderer must be flushed first, and when the re-init is completed the
123 // corresponding callback will be invoked to restart playback.
124 // The |stream| parameter specifies the new demuxer stream, and the |time|
125 // parameter specifies the time on media timeline where the switch occured.
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]126 void ReinitializeAudioRenderer(DemuxerStream* stream,
127 base::TimeDelta time,
128 base::OnceClosure reinitialize_completed_cb);
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]129 void OnAudioRendererReinitialized(DemuxerStream* stream,
130 base::TimeDelta time,
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]131 base::OnceClosure reinitialize_completed_cb,
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]132 PipelineStatus status);
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]133 void ReinitializeVideoRenderer(DemuxerStream* stream,
134 base::TimeDelta time,
135 base::OnceClosure restart_completed_cb);
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]136 void OnVideoRendererReinitialized(DemuxerStream* stream,
137 base::TimeDelta time,
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]138 base::OnceClosure restart_completed_cb,
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]139 PipelineStatus status);
140
141 // Restart audio/video renderer playback after a demuxer stream switch or
142 // after a demuxer stream has been disabled and re-enabled. The |stream|
143 // parameter specifies which stream needs to be restarted. The |time|
144 // parameter specifies the position on the media timeline where the playback
145 // needs to be restarted. It is necessary for demuxers with independent
146 // streams (e.g. MSE / ChunkDemuxer) to synchronize data reading between those
147 // streams.
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]148 void RestartAudioRenderer(DemuxerStream* stream,
149 base::TimeDelta time,
150 base::OnceClosure restart_completed_cb);
151 void RestartVideoRenderer(DemuxerStream* stream,
152 base::TimeDelta time,
153 base::OnceClosure restart_completed_cb);
154
155 // Fix state booleans after the stream switching is finished.
Daniel Cheng868ab272021-02-24 17:07:15[diff] [blame]156 void CleanUpTrackChange(base::OnceClosure on_finished,
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]157 bool* ended,
158 bool* playing);
servolkf25ceed2016-07-01 03:44:38[diff] [blame]159
xhwangbe9da702014-08-23 21:44:55[diff] [blame]160 // Callback executed by filters to update statistics.
alokp16bbeea2016-05-12 23:32:36[diff] [blame]161 void OnStatisticsUpdate(const PipelineStatistics& stats);
xhwangbe9da702014-08-23 21:44:55[diff] [blame]162
163 // Collection of callback methods and helpers for tracking changes in
164 // buffering state and transition from paused/underflow states and playing
165 // states.
166 //
167 // While in the kPlaying state:
168 // - A waiting to non-waiting transition indicates preroll has completed
169 // and StartPlayback() should be called
170 // - A non-waiting to waiting transition indicates underflow has occurred
171 // and PausePlayback() should be called
alokp16bbeea2016-05-12 23:32:36[diff] [blame]172 void OnBufferingStateChange(DemuxerStream::Type type,
Chris Cunninghamfc0d67e2019-07-22 20:29:16[diff] [blame]173 BufferingState new_buffering_state,
174 BufferingStateChangeReason reason);
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]175
servolkf25ceed2016-07-01 03:44:38[diff] [blame]176 // Handles the buffering notifications that we might get while an audio or a
177 // video stream is being restarted. In those cases we don't want to report
178 // underflows immediately and instead give decoders a chance to catch up with
179 // currently playing stream. Returns true if the buffering nofication has been
180 // handled and no further processing is necessary, returns false to indicate
181 // that we should fall back to the regular OnBufferingStateChange logic.
182 bool HandleRestartedStreamBufferingChanges(
183 DemuxerStream::Type type,
184 BufferingState new_buffering_state);
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]185
xhwangbe9da702014-08-23 21:44:55[diff] [blame]186 bool WaitingForEnoughData() const;
187 void PausePlayback();
188 void StartPlayback();
189
xhwangbe9da702014-08-23 21:44:55[diff] [blame]190 // Callbacks executed when a renderer has ended.
alokp16bbeea2016-05-12 23:32:36[diff] [blame]191 void OnRendererEnded(DemuxerStream::Type type);
scherkusece89452014-09-09 23:13:38[diff] [blame]192 bool PlaybackHasEnded() const;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]193 void RunEndedCallbackIfNeeded();
194
195 // Callback executed when a runtime error happens.
196 void OnError(PipelineStatus error);
Chris Cunningham038548b2017-07-10 22:36:30[diff] [blame]197
Ted Meyer3596315f2022-06-07 23:10:22[diff] [blame]198 // Callback executed when there is a fallback somewhere in the pipeline which
199 // should be recorded for metrics analysis.
200 void OnFallback(PipelineStatus fallback);
201
Xiaohan Wang640b41d2018-12-18 19:00:46[diff] [blame]202 void OnWaiting(WaitingReason reason);
alokp5d86e9b2016-05-17 20:20:41[diff] [blame]203 void OnVideoNaturalSizeChange(const gfx::Size& size);
Chris Cunningham038548b2017-07-10 22:36:30[diff] [blame]204 void OnAudioConfigChange(const AudioDecoderConfig& config);
205 void OnVideoConfigChange(const VideoDecoderConfig& config);
alokp5d86e9b2016-05-17 20:20:41[diff] [blame]206 void OnVideoOpacityChange(bool opaque);
Arthur Sonzogni581b9e82024-01-30 17:25:15[diff] [blame]207 void OnVideoFrameRateChange(std::optional<int> fps);
xhwangbe9da702014-08-23 21:44:55[diff] [blame]208
servolkbff54a22017-01-11 01:39:00[diff] [blame]209 void OnStreamRestartCompleted();
210
xhwangbe9da702014-08-23 21:44:55[diff] [blame]211 State state_;
212
213 // Task runner used to execute pipeline tasks.
Markus Handell2878995f2022-10-14 07:32:09[diff] [blame]214 scoped_refptr<base::SequencedTaskRunner> task_runner_;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]215
Keishi Hattori0e45c022021-11-27 09:25:52[diff] [blame]216 raw_ptr<MediaResource> media_resource_;
217 raw_ptr<RendererClient> client_;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]218
219 // Temporary callback used for Initialize() and Flush().
Chris Mumford84f095f2019-10-25 23:19:44[diff] [blame]220 PipelineStatusCallback init_cb_;
Chris Mumford1ccf4512019-10-25 19:34:05[diff] [blame]221 base::OnceClosure flush_cb_;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]222
alokp16bbeea2016-05-12 23:32:36[diff] [blame]223 std::unique_ptr<RendererClientInternal> audio_renderer_client_;
224 std::unique_ptr<RendererClientInternal> video_renderer_client_;
danakj4d43bc22016-04-26 03:36:04[diff] [blame]225 std::unique_ptr<AudioRenderer> audio_renderer_;
226 std::unique_ptr<VideoRenderer> video_renderer_;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]227
Keishi Hattori0e45c022021-11-27 09:25:52[diff] [blame]228 raw_ptr<DemuxerStream> current_audio_stream_;
229 raw_ptr<DemuxerStream> current_video_stream_;
servolk16e8bdf82017-04-11 17:00:39[diff] [blame]230
xhwangbe9da702014-08-23 21:44:55[diff] [blame]231 // Renderer-provided time source used to control playback.
Pârisdae06e02022-10-25 15:19:49[diff] [blame]232 raw_ptr<TimeSource, DanglingUntriaged> time_source_;
danakj4d43bc22016-04-26 03:36:04[diff] [blame]233 std::unique_ptr<WallClockTimeSource> wall_clock_time_source_;
scherkusece89452014-09-09 23:13:38[diff] [blame]234 bool time_ticking_;
a.berwal338bf002015-04-22 11:14:50[diff] [blame]235 double playback_rate_;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]236
237 // The time to start playback from after starting/seeking has completed.
238 base::TimeDelta start_time_;
239
240 BufferingState audio_buffering_state_;
241 BufferingState video_buffering_state_;
242
243 // Whether we've received the audio/video ended events.
244 bool audio_ended_;
245 bool video_ended_;
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]246 bool audio_playing_;
247 bool video_playing_;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]248
Keishi Hattori0e45c022021-11-27 09:25:52[diff] [blame]249 raw_ptr<CdmContext> cdm_context_;
xhwang97de4202014-11-25 08:44:01[diff] [blame]250
xhwangbe9da702014-08-23 21:44:55[diff] [blame]251 bool underflow_disabled_for_testing_;
scherkusece89452014-09-09 23:13:38[diff] [blame]252 bool clockless_video_playback_enabled_for_testing_;
xhwangbe9da702014-08-23 21:44:55[diff] [blame]253
dalecurtisc11e7bb2015-04-03 05:07:08[diff] [blame]254 // Used to defer underflow for video when audio is present.
Alex Turner3ae74622020-11-25 15:42:24[diff] [blame]255 base::CancelableOnceClosure deferred_video_underflow_cb_;
servolkf25ceed2016-07-01 03:44:38[diff] [blame]256
Alex Turner3ae74622020-11-25 15:42:24[diff] [blame]257 // We cannot use `!deferred_video_underflow_cb_.IsCancelled()` as that changes
258 // when the callback is run, even if not explicitly cancelled.
259 bool has_deferred_buffering_state_change_ = false;
dalecurtisc11e7bb2015-04-03 05:07:08[diff] [blame]260
261 // The amount of time to wait before declaring underflow if the video renderer
262 // runs out of data but the audio renderer still has enough.
liberato@chromium.org8705ffc2020-07-28 20:11:55[diff] [blame]263 Tuneable<base::TimeDelta> video_underflow_threshold_ = {
Peter Kastinge5a38ed2021-10-02 03:06:35[diff] [blame]264 "MediaVideoUnderflowThreshold", base::Milliseconds(1000),
265 base::Milliseconds(3000), base::Milliseconds(8000)};
dalecurtisc11e7bb2015-04-03 05:07:08[diff] [blame]266
servolk9dd6cc82017-05-24 05:39:02[diff] [blame]267 // Lock used to protect access to the |restarting_audio_| flag and
268 // |restarting_audio_time_|.
269 // TODO(servolk): Get rid of the lock and replace restarting_audio_ with
270 // std::atomic<bool> when atomics are unbanned in Chromium.
271 base::Lock restarting_audio_lock_;
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]272 bool pending_audio_track_change_ = false;
servolk9dd6cc82017-05-24 05:39:02[diff] [blame]273 base::TimeDelta restarting_audio_time_ = kNoTimestamp;
274
Ted Meyerda82b602018-04-19 00:36:56[diff] [blame]275 bool pending_video_track_change_ = false;
Dale Curtis3defa542017-11-07 00:52:32[diff] [blame]276
xhwangbe9da702014-08-23 21:44:55[diff] [blame]277 base::WeakPtr<RendererImpl> weak_this_;
Jeremy Roman32948392019-07-09 18:34:37[diff] [blame]278 base::WeakPtrFactory<RendererImpl> weak_factory_{this};
xhwangbe9da702014-08-23 21:44:55[diff] [blame]279};
280
281} // namespace media
282
servolkf54f5c8f2015-02-24 20:32:39[diff] [blame]283#endif // MEDIA_RENDERERS_RENDERER_IMPL_H_