From f6a7cceaaf461924e3f76cd3994bee748757d02a Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Tue, 15 Feb 2022 18:25:57 +0530 Subject: [PATCH 1/3] Add support for RTSP VP8 Added VP8 RTP packet reader and added support for VP8 playback through RTSP. Change-Id: Ie22ab79a234f61681cf95886a6ed8104a742f056 --- .../exoplayer/rtsp/RtpPayloadFormat.java | 4 + .../media3/exoplayer/rtsp/RtspMediaTrack.java | 8 + .../DefaultRtpPayloadReaderFactory.java | 2 + .../exoplayer/rtsp/reader/RtpVP8Reader.java | 220 ++++++++++++++++++ 4 files changed, 234 insertions(+) create mode 100644 libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 297353167b9..6d1f5fc953b 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -40,6 +40,7 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; private static final String RTP_MEDIA_H264 = "H264"; private static final String RTP_MEDIA_H265 = "H265"; + private static final String RTP_MEDIA_VP8 = "VP8"; /** Returns whether the format of a {@link MediaDescription} is supported. */ public static boolean isFormatSupported(MediaDescription mediaDescription) { @@ -48,6 +49,7 @@ public static boolean isFormatSupported(MediaDescription mediaDescription) { case RTP_MEDIA_H264: case RTP_MEDIA_H265: case RTP_MEDIA_MPEG4_GENERIC: + case RTP_MEDIA_VP8: return true; default: return false; @@ -71,6 +73,8 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { return MimeTypes.VIDEO_H265; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; + case RTP_MEDIA_VP8: + return MimeTypes.VIDEO_VP8; default: throw new IllegalArgumentException(mediaType); } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 7547f1ea188..3f1b297bcf6 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -56,6 +56,10 @@ private static final String GENERIC_CONTROL_ATTR = "*"; + /** Default width and height for VP8. */ + private static final int DEFAULT_VP8_WIDTH = 320; + private static final int DEFAULT_VP8_HEIGHT = 240; + /** The track's associated {@link RtpPayloadFormat}. */ public final RtpPayloadFormat payloadFormat; /** The track's URI. */ @@ -129,6 +133,10 @@ public int hashCode() { checkArgument(!fmtpParameters.isEmpty()); processH265FmtpAttribute(formatBuilder, fmtpParameters); break; + case MimeTypes.VIDEO_VP8: + // VP8 does not require a FMTP attribute. So Setting default width and height. + formatBuilder.setWidth(DEFAULT_VP8_WIDTH).setHeight(DEFAULT_VP8_HEIGHT); + break; case MimeTypes.AUDIO_AC3: // AC3 does not require a FMTP attribute. Fall through. default: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 888939b7e89..ed57160db60 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -40,6 +40,8 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) { return new RtpH264Reader(payloadFormat); case MimeTypes.VIDEO_H265: return new RtpH265Reader(payloadFormat); + case MimeTypes.VIDEO_VP8: + return new RtpVP8Reader(payloadFormat); default: // No supported reader, returning null. } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java new file mode 100644 index 00000000000..3adbca1b73f --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java @@ -0,0 +1,220 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.castNonNull; + +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.TrackOutput; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses a VP8 byte stream carried on RTP packets, and extracts VP8 individual video frames as + * defined in RFC7741. + */ +/* package */ final class RtpVP8Reader implements RtpPayloadReader { + private static final String TAG = "RtpVP8Reader"; + + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + @C.BufferFlags private int bufferFlags; + + private long firstReceivedTimestamp; + private int previousSequenceNumber; + /** The combined size of a sample that is fragmented into multiple RTP packets. */ + private int fragmentedSampleSizeBytes; + private long startTimeOffsetUs; + private boolean gotFirstPacketOfVP8Frame; + private boolean isKeyFrame; + private boolean isOutputFormatSet; + + /** Creates an instance. */ + public RtpVP8Reader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + firstReceivedTimestamp = C.TIME_UNSET; + previousSequenceNumber = C.INDEX_UNSET; + fragmentedSampleSizeBytes = 0; + gotFirstPacketOfVP8Frame = false; + isKeyFrame = false; + isOutputFormatSet = false; + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); + castNonNull(trackOutput).format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} + + @Override + public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) + throws ParserException { + checkStateNotNull(trackOutput); + + if (parseVP8Descriptor(data, sequenceNumber)) { + // VP8 Payload Header, RFC7741 Section 4.3 + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |Size0|H| VER |P| + // +-+-+-+-+-+-+-+-+ + // P: Inverse key frame flag. + if (fragmentedSampleSizeBytes == 0 && gotFirstPacketOfVP8Frame) { + isKeyFrame = (data.peekUnsignedByte() & 0x01) == 0; + } + if (!isOutputFormatSet) { + // Parsing frame data to get width and height, RFC6386 Section 9.1 + int currPosition = data.getPosition(); + data.setPosition(currPosition + 6); + int width = data.readLittleEndianUnsignedShort() & 0x3fff; + int height = data.readLittleEndianUnsignedShort() & 0x3fff; + data.setPosition(currPosition); + + if (width != payloadFormat.format.width || height != payloadFormat.format.height) { + Format.Builder formatBuilder = new Format.Builder(); + if (payloadFormat.format.bitrate > 0) { + formatBuilder.setAverageBitrate(payloadFormat.format.bitrate); + } + formatBuilder.setSampleMimeType(payloadFormat.format.sampleMimeType); + formatBuilder.setWidth(width).setHeight(height); + trackOutput.format(formatBuilder.build()); + } + isOutputFormatSet = true; + } + + int fragmentSize = data.bytesLeft(); + // Write the video sample + trackOutput.sampleData(data, fragmentSize); + fragmentedSampleSizeBytes += fragmentSize; + + if (rtpMarker) { + if (firstReceivedTimestamp == C.TIME_UNSET) { + firstReceivedTimestamp = timestamp; + } + bufferFlags = isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0; + long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); + trackOutput.sampleMetadata( + timeUs, + bufferFlags, + fragmentedSampleSizeBytes, + /* offset= */ 0, + /* encryptionData= */ null); + fragmentedSampleSizeBytes = 0; + gotFirstPacketOfVP8Frame = false; + } + previousSequenceNumber = sequenceNumber; + } + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + fragmentedSampleSizeBytes = 0; + startTimeOffsetUs = timeUs; + } + + // Internal methods. + private boolean parseVP8Descriptor(ParsableByteArray payload, int packetSequenceNumber) { + // VP8 Payload Descriptor, RFC7741 Section 4.2 + // 0 1 2 3 4 5 6 7 + // +-+-+-+-+-+-+-+-+ + // |X|R|N|S|R| PID | (REQUIRED) + // +-+-+-+-+-+-+-+-+ + // X: |I|L|T|K| RSV | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ + // I: |M| PictureID | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ + // L: | TL0PICIDX | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ + // T/K: |TID|Y| KEYIDX | (OPTIONAL) + // +-+-+-+-+-+-+-+-+ + + int header = payload.readUnsignedByte(); + if (!gotFirstPacketOfVP8Frame) { + // For start of VP8 partition S=1 and PID=0 as per RFC7741 Section 4.2 + if ((header & 0x17) != 0x10) { + Log.w( + TAG, + Util.formatInvariant( + "first payload octet of the RTP packet is not the beginning of a new VP8 " + + "partition, Dropping current packet")); + return false; + } + gotFirstPacketOfVP8Frame = true; + } else { + // Check that this packet is in the sequence of the previous packet. + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (packetSequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d." + + " Dropping packet.", + expectedSequenceNumber, packetSequenceNumber)); + return false; + } + } + + // Check if optional X header is present + if ((header & 0x80) != 0) { + int xHeader = payload.readUnsignedByte(); + + // Check if optional I header present + if ((xHeader & 0x80) != 0) { + int iHeader = payload.readUnsignedByte(); + if ((iHeader & 0x80) != 0) { + payload.skipBytes(1); + Log.i(TAG, "15 bits PictureID"); + } else { + Log.i(TAG, "7 bits PictureID"); + } + } + + // Check if optional L header present + if ((xHeader & 0x40) != 0) { + payload.skipBytes(1); + } + + // Check if optional T or K header(s) present + if ((xHeader & 0x20) != 0 || (xHeader & 0x10) != 0) { + payload.skipBytes(1); + } + } + return true; + } + + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } +} From f2e0953643dce81666c86d4e7ffb6b3dd1ffb16d Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Thu, 3 Mar 2022 19:33:21 +0530 Subject: [PATCH 2/3] Updated way to create a formatBuilder Change-Id: I2c8eb8d6ee28d8c044d71db042f3b186ea5762f3 --- .../media3/exoplayer/rtsp/reader/RtpVP8Reader.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java index 3adbca1b73f..d7860378335 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java @@ -97,11 +97,8 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, data.setPosition(currPosition); if (width != payloadFormat.format.width || height != payloadFormat.format.height) { - Format.Builder formatBuilder = new Format.Builder(); - if (payloadFormat.format.bitrate > 0) { - formatBuilder.setAverageBitrate(payloadFormat.format.bitrate); - } - formatBuilder.setSampleMimeType(payloadFormat.format.sampleMimeType); + Format trackFormat = payloadFormat.format; + Format.Builder formatBuilder = trackFormat.buildUpon(); formatBuilder.setWidth(width).setHeight(height); trackOutput.format(formatBuilder.build()); } From 8afa7a548afd435ec7f4596a826ed8b8203abe7b Mon Sep 17 00:00:00 2001 From: Rakesh Kumar Date: Tue, 8 Mar 2022 16:54:36 +0530 Subject: [PATCH 3/3] Fix review comments in RtpVP8Reader Change-Id: Id47c746b199831d0bb51dc736c43fd20c2e79c08 --- .../exoplayer/rtsp/reader/RtpVP8Reader.java | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java index d7860378335..e2ffed7142e 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpVP8Reader.java @@ -42,7 +42,6 @@ private final RtpPayloadFormat payloadFormat; private @MonotonicNonNull TrackOutput trackOutput; - @C.BufferFlags private int bufferFlags; private long firstReceivedTimestamp; private int previousSequenceNumber; @@ -78,7 +77,9 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, throws ParserException { checkStateNotNull(trackOutput); - if (parseVP8Descriptor(data, sequenceNumber)) { + // Check if valid VP8 Payload Descriptor is present + boolean isValidVP8Descriptor = parseVP8Descriptor(data, sequenceNumber); + if (isValidVP8Descriptor) { // VP8 Payload Header, RFC7741 Section 4.3 // 0 1 2 3 4 5 6 7 // +-+-+-+-+-+-+-+-+ @@ -89,7 +90,7 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, isKeyFrame = (data.peekUnsignedByte() & 0x01) == 0; } if (!isOutputFormatSet) { - // Parsing frame data to get width and height, RFC6386 Section 9.1 + // Parsing frame data to get width and height, RFC6386 Section 19.1 int currPosition = data.getPosition(); data.setPosition(currPosition + 6); int width = data.readLittleEndianUnsignedShort() & 0x3fff; @@ -97,10 +98,8 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, data.setPosition(currPosition); if (width != payloadFormat.format.width || height != payloadFormat.format.height) { - Format trackFormat = payloadFormat.format; - Format.Builder formatBuilder = trackFormat.buildUpon(); - formatBuilder.setWidth(width).setHeight(height); - trackOutput.format(formatBuilder.build()); + trackOutput.format( + payloadFormat.format.buildUpon().setWidth(width).setHeight(height).build()); } isOutputFormatSet = true; } @@ -114,11 +113,10 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, if (firstReceivedTimestamp == C.TIME_UNSET) { firstReceivedTimestamp = timestamp; } - bufferFlags = isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0; long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); trackOutput.sampleMetadata( timeUs, - bufferFlags, + isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0, fragmentedSampleSizeBytes, /* offset= */ 0, /* encryptionData= */ null); @@ -156,11 +154,8 @@ private boolean parseVP8Descriptor(ParsableByteArray payload, int packetSequence if (!gotFirstPacketOfVP8Frame) { // For start of VP8 partition S=1 and PID=0 as per RFC7741 Section 4.2 if ((header & 0x17) != 0x10) { - Log.w( - TAG, - Util.formatInvariant( - "first payload octet of the RTP packet is not the beginning of a new VP8 " - + "partition, Dropping current packet")); + Log.w(TAG,"first payload octet of the RTP packet is not the beginning of a new VP8 " + + "partition, Dropping current packet"); return false; } gotFirstPacketOfVP8Frame = true; @@ -187,9 +182,6 @@ private boolean parseVP8Descriptor(ParsableByteArray payload, int packetSequence int iHeader = payload.readUnsignedByte(); if ((iHeader & 0x80) != 0) { payload.skipBytes(1); - Log.i(TAG, "15 bits PictureID"); - } else { - Log.i(TAG, "7 bits PictureID"); } }