Schedule doSomeWork when MediaCodec signals available buffers

When running in asynchronous mode, MediaCodec will be running the CPU to signal input and output buffers being made available for use by the player. With ExoPlayer.experimentalSetDynamicSchedulingEnabled set to true, ExoPlayer will wakeup to make rendering progress when MediaCodec raises these signals. In this way, ExoPlayer work will align more closely with CPU wake-cycles. PiperOrigin-RevId: 638962108
androidx · May 31, 2024 · ac34798 · ac34798
1 parent 1329821
commit ac34798
Show file tree

Hide file tree

Showing 7 changed files with 190 additions and 7 deletions.
diff --git a/RELEASENOTES.md b/RELEASENOTES.md
@@ -50,6 +50,12 @@
         order for the renderer to progress. If `ExoPlayer` is set with
         `experimentalSetDynamicSchedulingEnabled` then `ExoPlayer` will call
         this method when calculating the time to schedule its work task.
+    *   Add `MediaCodecAdapter#OnBufferAvailableListener` to alert when input
+        and output buffers are available for use by `MediaCodecRenderer`.
+        `MediaCodecRenderer` will signal `ExoPlayer` when receiving these
+        callbacks and if `ExoPlayer` is set with
+        `experimentalSetDynamicSchedulingEnabled`, then `ExoPlayer` will
+        schedule its work loop as renderers can make progress.
 *   Transformer:
     *   Work around a decoder bug where the number of audio channels was capped
         at stereo when handling PCM input.

diff --git a/...ries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java b/...ries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java
@@ -119,8 +119,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   private long currentPositionUs;
   private boolean allowPositionDiscontinuity;
   private boolean audioSinkNeedsReset;
-
-  @Nullable private WakeupListener wakeupListener;
   private boolean hasPendingReportedSkippedSilence;
   private int rendererPriority;
   private boolean isStarted;
@@ -480,7 +478,8 @@ public MediaClock getMediaClock() {
   }
 
   @Override
-  public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) {
+  public long getDurationToProgressUs(
+      boolean isOnBufferAvailableListenerRegistered, long positionUs, long elapsedRealtimeUs) {
     if (nextBufferToWritePresentationTimeUs != C.TIME_UNSET) {
       long durationUs =
           (long)
@@ -493,7 +492,8 @@ public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) {
       }
       return max(DEFAULT_DURATION_TO_PROGRESS_US, durationUs);
     }
-    return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs);
+    return super.getDurationToProgressUs(
+        isOnBufferAvailableListenerRegistered, positionUs, elapsedRealtimeUs);
   }
 
   @Override
@@ -854,9 +854,6 @@ public void handleMessage(@MessageType int messageType, @Nullable Object message
       case MSG_SET_AUDIO_SESSION_ID:
         audioSink.setAudioSessionId((Integer) checkNotNull(message));
         break;
-      case MSG_SET_WAKEUP_LISTENER:
-        this.wakeupListener = (WakeupListener) message;
-        break;
       case MSG_SET_PRIORITY:
         rendererPriority = (int) checkNotNull(message);
         updateCodecImportance();
@@ -1073,13 +1070,15 @@ public void onSkipSilenceEnabledChanged(boolean skipSilenceEnabled) {
 
     @Override
     public void onOffloadBufferEmptying() {
+      WakeupListener wakeupListener = getWakeupListener();
       if (wakeupListener != null) {
         wakeupListener.onWakeup();
       }
     }
 
     @Override
     public void onOffloadBufferFull() {
+      WakeupListener wakeupListener = getWakeupListener();
       if (wakeupListener != null) {
         wakeupListener.onSleep();
       }

diff --git a/...yer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java b/...yer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java
@@ -274,6 +274,12 @@ public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler
         handler);
   }
 
+  @Override
+  public boolean registerOnBufferAvailableListener(OnBufferAvailableListener listener) {
+    asynchronousMediaCodecCallback.setOnBufferAvailableListener(listener);
+    return true;
+  }
+
   @Override
   public void setOutputSurface(Surface surface) {
     codec.setOutputSurface(surface);

diff --git a/...er/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java b/...er/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallback.java
@@ -77,6 +77,10 @@
   @Nullable
   private IllegalStateException internalException;
 
+  @GuardedBy("lock")
+  @Nullable
+  private MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener;
+
   /**
    * Creates a new instance.
    *
@@ -210,6 +214,9 @@ public void flush() {
   public void onInputBufferAvailable(MediaCodec codec, int index) {
     synchronized (lock) {
       availableInputBuffers.addLast(index);
+      if (onBufferAvailableListener != null) {
+        onBufferAvailableListener.onInputBufferAvailable();
+      }
     }
   }
 
@@ -222,6 +229,9 @@ public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.Buff
       }
       availableOutputBuffers.addLast(index);
       bufferInfos.add(info);
+      if (onBufferAvailableListener != null) {
+        onBufferAvailableListener.onOutputBufferAvailable();
+      }
     }
   }
 
@@ -247,6 +257,20 @@ public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
     }
   }
 
+  /**
+   * Sets the {@link MediaCodecAdapter.OnBufferAvailableListener} that will be notified when {@link
+   * #onInputBufferAvailable} and {@link #onOutputBufferAvailable} are called.
+   *
+   * @param onBufferAvailableListener The listener that will be notified when {@link
+   *     #onInputBufferAvailable} and {@link #onOutputBufferAvailable} are called.
+   */
+  public void setOnBufferAvailableListener(
+      MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener) {
+    synchronized (lock) {
+      this.onBufferAvailableListener = onBufferAvailableListener;
+    }
+  }
+
   private void onFlushCompleted() {
     synchronized (lock) {
       if (shutDown) {

diff --git a/...aries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java b/...aries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java
@@ -151,6 +151,23 @@ interface OnFrameRenderedListener {
     void onFrameRendered(MediaCodecAdapter codec, long presentationTimeUs, long nanoTime);
   }
 
+  /** Listener to be called when an input or output buffer becomes available. */
+  interface OnBufferAvailableListener {
+    /**
+     * Called when an input buffer becomes available.
+     *
+     * @see MediaCodec.Callback#onInputBufferAvailable(MediaCodec, int)
+     */
+    default void onInputBufferAvailable() {}
+
+    /**
+     * Called when an output buffer becomes available.
+     *
+     * @see MediaCodec.Callback#onOutputBufferAvailable(MediaCodec, int, MediaCodec.BufferInfo)
+     */
+    default void onOutputBufferAvailable() {}
+  }
+
   /**
    * Returns the next available input buffer index from the underlying {@link MediaCodec} or {@link
    * MediaCodec#INFO_TRY_AGAIN_LATER} if no such buffer exists.
@@ -252,6 +269,21 @@ void queueSecureInputBuffer(
   @RequiresApi(23)
   void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler);
 
+  /**
+   * Registers a listener that will be called when an input or output buffer becomes available.
+   *
+   * <p>Returns false if listener was not successfully registered for callbacks.
+   *
+   * @see MediaCodec.Callback#onInputBufferAvailable
+   * @see MediaCodec.Callback#onOutputBufferAvailable
+   * @return Whether listener was successfully registered.
+   */
+  @RequiresApi(21)
+  default boolean registerOnBufferAvailableListener(
+      MediaCodecAdapter.OnBufferAvailableListener listener) {
+    return false;
+  }
+
   /**
    * Dynamically sets the output surface of a {@link MediaCodec}.
    *

diff --git a/...ries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/...ries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
@@ -346,6 +346,7 @@ private static String buildCustomDiagnosticInfo(int errorCode) {
   @Nullable private Format outputFormat;
   @Nullable private DrmSession codecDrmSession;
   @Nullable private DrmSession sourceDrmSession;
+  @Nullable private WakeupListener wakeupListener;
 
   /**
    * A framework {@link MediaCrypto} for use with {@link MediaCodec#queueSecureInputBuffer(int, int,
@@ -382,6 +383,7 @@ private static String buildCustomDiagnosticInfo(int errorCode) {
   private boolean codecNeedsAdaptationWorkaroundBuffer;
   private boolean shouldSkipAdaptationWorkaroundOutputBuffer;
   private boolean codecNeedsEosPropagation;
+  private boolean codecRegisteredOnBufferAvailableListener;
   private long codecHotswapDeadlineMs;
   private int inputIndex;
   private int outputIndex;
@@ -503,6 +505,37 @@ public void setRenderTimeLimitMs(long renderTimeLimitMs) {
   protected abstract @Capabilities int supportsFormat(
       MediaCodecSelector mediaCodecSelector, Format format) throws DecoderQueryException;
 
+  @Override
+  public final long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) {
+    return getDurationToProgressUs(
+        /* isOnBufferAvailableListenerRegistered= */ codecRegisteredOnBufferAvailableListener,
+        positionUs,
+        elapsedRealtimeUs);
+  }
+
+  /**
+   * Returns minimum time playback must advance in order for the {@link #render} call to make
+   * progress.
+   *
+   * <p>If the {@code Renderer} has a registered {@link
+   * MediaCodecAdapter.OnBufferAvailableListener}, then the {@code Renderer} will be notified when
+   * decoder input and output buffers become available. These callbacks may affect the calculated
+   * minimum time playback must advance before a {@link #render} call can make progress.
+   *
+   * @param isOnBufferAvailableListenerRegistered Whether the {@code Renderer} is using a {@link
+   *     MediaCodecAdapter} with successfully registered {@link
+   *     MediaCodecAdapter.OnBufferAvailableListener OnBufferAvailableListener}.
+   * @param positionUs The current media time in microseconds, measured at the start of the current
+   *     iteration of the rendering loop.
+   * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
+   *     measured at the start of the current iteration of the rendering loop.
+   * @return minimum time playback must advance before renderer is able to make progress.
+   */
+  protected long getDurationToProgressUs(
+      boolean isOnBufferAvailableListenerRegistered, long positionUs, long elapsedRealtimeUs) {
+    return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs);
+  }
+
   /**
    * Returns a list of decoders that can decode media in the specified format, in priority order.
    *
@@ -797,6 +830,16 @@ protected void onStopped() {
     // Do nothing. Overridden to remove throws clause.
   }
 
+  @Override
+  public void handleMessage(@MessageType int messageType, @Nullable Object message)
+      throws ExoPlaybackException {
+    if (messageType == MSG_SET_WAKEUP_LISTENER) {
+      this.wakeupListener = (WakeupListener) message;
+    } else {
+      super.handleMessage(messageType, message);
+    }
+  }
+
   @Override
   public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
     if (pendingOutputEndOfStream) {
@@ -971,6 +1014,7 @@ protected void resetCodecStateForRelease() {
     codecNeedsEosBufferTimestampWorkaround = false;
     codecNeedsMonoChannelCountWorkaround = false;
     codecNeedsEosPropagation = false;
+    codecRegisteredOnBufferAvailableListener = false;
     codecReconfigured = false;
     codecReconfigurationState = RECONFIGURATION_STATE_NONE;
   }
@@ -1193,6 +1237,10 @@ private void initCodec(MediaCodecInfo codecInfo, @Nullable MediaCrypto crypto) t
     try {
       TraceUtil.beginSection("createCodec:" + codecName);
       codec = codecAdapterFactory.createAdapter(configuration);
+      codecRegisteredOnBufferAvailableListener =
+          Util.SDK_INT >= 21
+              && Api21.registerOnBufferAvailableListener(
+                  codec, new MediaCodecRendererCodecAdapterListener());
     } finally {
       TraceUtil.endSection();
     }
@@ -1813,6 +1861,12 @@ protected float getCodecOperatingRateV23(
     return CODEC_OPERATING_RATE_UNSET;
   }
 
+  /** Returns listener used to signal that {@link #render(long, long)} should be called. */
+  @Nullable
+  protected final WakeupListener getWakeupListener() {
+    return wakeupListener;
+  }
+
   /**
    * Updates the codec operating rate, or triggers codec release and re-initialization if a
    * previously set operating rate needs to be cleared.
@@ -2691,6 +2745,15 @@ public OutputStreamInfo(
     }
   }
 
+  @RequiresApi(21)
+  private static final class Api21 {
+    @DoNotInline
+    public static boolean registerOnBufferAvailableListener(
+        MediaCodecAdapter codec, MediaCodecRendererCodecAdapterListener listener) {
+      return codec.registerOnBufferAvailableListener(listener);
+    }
+  }
+
   @RequiresApi(31)
   private static final class Api31 {
     private Api31() {}
@@ -2704,4 +2767,21 @@ public static void setLogSessionIdToMediaCodecFormat(
       }
     }
   }
+
+  private final class MediaCodecRendererCodecAdapterListener
+      implements MediaCodecAdapter.OnBufferAvailableListener {
+    @Override
+    public void onInputBufferAvailable() {
+      if (wakeupListener != null) {
+        wakeupListener.onWakeup();
+      }
+    }
+
+    @Override
+    public void onOutputBufferAvailable() {
+      if (wakeupListener != null) {
+        wakeupListener.onWakeup();
+      }
+    }
+  }
 }
diff --git a/...rc/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallbackTest.java b/...rc/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecCallbackTest.java
@@ -530,6 +530,42 @@ public void shutdown_withPendingError_doesNotThrow() throws Exception {
     asynchronousMediaCodecCallback.shutdown();
   }
 
+  @Test
+  public void onInputBufferAvailable_withOnBufferAvailableListener_callsOnInputBufferAvailable() {
+    AtomicInteger onInputBufferAvailableCounter = new AtomicInteger();
+    MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener =
+        new MediaCodecAdapter.OnBufferAvailableListener() {
+          @Override
+          public void onInputBufferAvailable() {
+            onInputBufferAvailableCounter.getAndIncrement();
+          }
+        };
+    asynchronousMediaCodecCallback.setOnBufferAvailableListener(onBufferAvailableListener);
+
+    // Send an input buffer to the callback.
+    asynchronousMediaCodecCallback.onInputBufferAvailable(codec, 0);
+
+    assertThat(onInputBufferAvailableCounter.get()).isEqualTo(1);
+  }
+
+  @Test
+  public void onOutputBufferAvailable_withOnBufferAvailableListener_callsOnOutputBufferAvailable() {
+    AtomicInteger onOutputBufferAvailableCounter = new AtomicInteger();
+    MediaCodecAdapter.OnBufferAvailableListener onBufferAvailableListener =
+        new MediaCodecAdapter.OnBufferAvailableListener() {
+          @Override
+          public void onOutputBufferAvailable() {
+            onOutputBufferAvailableCounter.getAndIncrement();
+          }
+        };
+    asynchronousMediaCodecCallback.setOnBufferAvailableListener(onBufferAvailableListener);
+
+    // Send an output buffer to the callback.
+    asynchronousMediaCodecCallback.onOutputBufferAvailable(codec, 0, new MediaCodec.BufferInfo());
+
+    assertThat(onOutputBufferAvailableCounter.get()).isEqualTo(1);
+  }
+
   /** Reflectively create a {@link MediaCodec.CodecException}. */
   private static MediaCodec.CodecException createCodecException()
       throws NoSuchMethodException,