Add support for RTSP VP8 #47
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,217 @@ | ||
| /* | ||
| * Copyright 2022 The Android Open Source Project | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package androidx.media3.exoplayer.rtsp.reader; | ||
|
|
||
| import static androidx.media3.common.util.Assertions.checkStateNotNull; | ||
| import static androidx.media3.common.util.Util.castNonNull; | ||
|
|
||
| import androidx.media3.common.C; | ||
| import androidx.media3.common.Format; | ||
| import androidx.media3.common.ParserException; | ||
| import androidx.media3.common.util.Log; | ||
| import androidx.media3.common.util.ParsableByteArray; | ||
| import androidx.media3.common.util.Util; | ||
| import androidx.media3.exoplayer.rtsp.RtpPacket; | ||
| import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; | ||
| import androidx.media3.extractor.ExtractorOutput; | ||
| import androidx.media3.extractor.TrackOutput; | ||
| import org.checkerframework.checker.nullness.qual.MonotonicNonNull; | ||
|
|
||
| /** | ||
| * Parses a VP8 byte stream carried on RTP packets, and extracts VP8 individual video frames as | ||
| * defined in RFC7741. | ||
| */ | ||
| /* package */ final class RtpVP8Reader implements RtpPayloadReader { | ||
| private static final String TAG = "RtpVP8Reader"; | ||
|
|
||
| private static final long MEDIA_CLOCK_FREQUENCY = 90_000; | ||
|
|
||
| private final RtpPayloadFormat payloadFormat; | ||
|
|
||
| private @MonotonicNonNull TrackOutput trackOutput; | ||
| @C.BufferFlags private int bufferFlags; | ||
|
|
||
| private long firstReceivedTimestamp; | ||
| private int previousSequenceNumber; | ||
| /** The combined size of a sample that is fragmented into multiple RTP packets. */ | ||
| private int fragmentedSampleSizeBytes; | ||
| private long startTimeOffsetUs; | ||
| private boolean gotFirstPacketOfVP8Frame; | ||
| private boolean isKeyFrame; | ||
| private boolean isOutputFormatSet; | ||
|
|
||
| /** Creates an instance. */ | ||
| public RtpVP8Reader(RtpPayloadFormat payloadFormat) { | ||
| this.payloadFormat = payloadFormat; | ||
|
There was a problem hiding this comment. please order these to the declaration order. Also initialize There was a problem hiding this comment. Can you elaborate on the order you want to keep because as for my knowledge variables are in the same order of declaration. |
||
| firstReceivedTimestamp = C.TIME_UNSET; | ||
| previousSequenceNumber = C.INDEX_UNSET; | ||
| fragmentedSampleSizeBytes = 0; | ||
| gotFirstPacketOfVP8Frame = false; | ||
|
There was a problem hiding this comment.
There was a problem hiding this comment. In VP8 a frame can be divided into partitions, these partitions can be sent as a packet. This variable is used to check that the packet received is the first packet of a frame. |
||
| isKeyFrame = false; | ||
| isOutputFormatSet = false; | ||
| } | ||
|
|
||
| @Override | ||
| public void createTracks(ExtractorOutput extractorOutput, int trackId) { | ||
| trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); | ||
| castNonNull(trackOutput).format(payloadFormat.format); | ||
| } | ||
|
|
||
| @Override | ||
| public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} | ||
|
|
||
| @Override | ||
| public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) | ||
| throws ParserException { | ||
| checkStateNotNull(trackOutput); | ||
|
|
||
| if (parseVP8Descriptor(data, sequenceNumber)) { | ||
rakeshnitb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| // VP8 Payload Header, RFC7741 Section 4.3 | ||
| // 0 1 2 3 4 5 6 7 | ||
| // +-+-+-+-+-+-+-+-+ | ||
| // |Size0|H| VER |P| | ||
| // +-+-+-+-+-+-+-+-+ | ||
| // P: Inverse key frame flag. | ||
| if (fragmentedSampleSizeBytes == 0 && gotFirstPacketOfVP8Frame) { | ||
| isKeyFrame = (data.peekUnsignedByte() & 0x01) == 0; | ||
| } | ||
| if (!isOutputFormatSet) { | ||
| // Parsing frame data to get width and height, RFC6386 Section 9.1 | ||
| int currPosition = data.getPosition(); | ||
| data.setPosition(currPosition + 6); | ||
| int width = data.readLittleEndianUnsignedShort() & 0x3fff; | ||
|
There was a problem hiding this comment. Do we need littleEndian here? I assume the data here is in "network order". Also this library could be used on big-endian devices, will using littleEndian method here limit the usecases? There was a problem hiding this comment. According to RFC6386 Section 19.1 width and height are save in littleEndianess. |
||
| int height = data.readLittleEndianUnsignedShort() & 0x3fff; | ||
| data.setPosition(currPosition); | ||
|
|
||
| if (width != payloadFormat.format.width || height != payloadFormat.format.height) { | ||
| Format trackFormat = payloadFormat.format; | ||
| Format.Builder formatBuilder = trackFormat.buildUpon(); | ||
rakeshnitb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| formatBuilder.setWidth(width).setHeight(height); | ||
| trackOutput.format(formatBuilder.build()); | ||
|
There was a problem hiding this comment. Double checking: If playing back a 1080p VP8 video, the player is able to pick up the 1080p format information? Because no other RTP readers reconfigures track format. There was a problem hiding this comment. I have checked that the player is able to pick format information for 1080p. |
||
| } | ||
| isOutputFormatSet = true; | ||
| } | ||
|
|
||
| int fragmentSize = data.bytesLeft(); | ||
| // Write the video sample | ||
| trackOutput.sampleData(data, fragmentSize); | ||
| fragmentedSampleSizeBytes += fragmentSize; | ||
|
|
||
| if (rtpMarker) { | ||
| if (firstReceivedTimestamp == C.TIME_UNSET) { | ||
| firstReceivedTimestamp = timestamp; | ||
| } | ||
| bufferFlags = isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0; | ||
rakeshnitb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); | ||
| trackOutput.sampleMetadata( | ||
| timeUs, | ||
| bufferFlags, | ||
| fragmentedSampleSizeBytes, | ||
| /* offset= */ 0, | ||
| /* encryptionData= */ null); | ||
| fragmentedSampleSizeBytes = 0; | ||
| gotFirstPacketOfVP8Frame = false; | ||
| } | ||
| previousSequenceNumber = sequenceNumber; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void seek(long nextRtpTimestamp, long timeUs) { | ||
| firstReceivedTimestamp = nextRtpTimestamp; | ||
| fragmentedSampleSizeBytes = 0; | ||
| startTimeOffsetUs = timeUs; | ||
| } | ||
|
|
||
| // Internal methods. | ||
| private boolean parseVP8Descriptor(ParsableByteArray payload, int packetSequenceNumber) { | ||
| // VP8 Payload Descriptor, RFC7741 Section 4.2 | ||
| // 0 1 2 3 4 5 6 7 | ||
| // +-+-+-+-+-+-+-+-+ | ||
| // |X|R|N|S|R| PID | (REQUIRED) | ||
| // +-+-+-+-+-+-+-+-+ | ||
| // X: |I|L|T|K| RSV | (OPTIONAL) | ||
| // +-+-+-+-+-+-+-+-+ | ||
| // I: |M| PictureID | (OPTIONAL) | ||
| // +-+-+-+-+-+-+-+-+ | ||
| // L: | TL0PICIDX | (OPTIONAL) | ||
| // +-+-+-+-+-+-+-+-+ | ||
| // T/K: |TID|Y| KEYIDX | (OPTIONAL) | ||
| // +-+-+-+-+-+-+-+-+ | ||
|
|
||
| int header = payload.readUnsignedByte(); | ||
| if (!gotFirstPacketOfVP8Frame) { | ||
| // For start of VP8 partition S=1 and PID=0 as per RFC7741 Section 4.2 | ||
| if ((header & 0x17) != 0x10) { | ||
| Log.w( | ||
| TAG, | ||
| Util.formatInvariant( | ||
rakeshnitb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| "first payload octet of the RTP packet is not the beginning of a new VP8 " | ||
| + "partition, Dropping current packet")); | ||
| return false; | ||
| } | ||
| gotFirstPacketOfVP8Frame = true; | ||
| } else { | ||
| // Check that this packet is in the sequence of the previous packet. | ||
| int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); | ||
| if (packetSequenceNumber != expectedSequenceNumber) { | ||
| Log.w( | ||
| TAG, | ||
| Util.formatInvariant( | ||
| "Received RTP packet with unexpected sequence number. Expected: %d; received: %d." | ||
| + " Dropping packet.", | ||
| expectedSequenceNumber, packetSequenceNumber)); | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| // Check if optional X header is present | ||
| if ((header & 0x80) != 0) { | ||
| int xHeader = payload.readUnsignedByte(); | ||
|
|
||
| // Check if optional I header present | ||
| if ((xHeader & 0x80) != 0) { | ||
| int iHeader = payload.readUnsignedByte(); | ||
| if ((iHeader & 0x80) != 0) { | ||
| payload.skipBytes(1); | ||
| Log.i(TAG, "15 bits PictureID"); | ||
rakeshnitb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } else { | ||
| Log.i(TAG, "7 bits PictureID"); | ||
| } | ||
| } | ||
|
|
||
| // Check if optional L header present | ||
| if ((xHeader & 0x40) != 0) { | ||
| payload.skipBytes(1); | ||
| } | ||
|
|
||
| // Check if optional T or K header(s) present | ||
| if ((xHeader & 0x20) != 0 || (xHeader & 0x10) != 0) { | ||
| payload.skipBytes(1); | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
|
|
||
| private static long toSampleUs( | ||
| long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { | ||
| return startTimeOffsetUs | ||
| + Util.scaleLargeTimestamp( | ||
| (rtpTimestamp - firstReceivedRtpTimestamp), | ||
| /* multiplier= */ C.MICROS_PER_SECOND, | ||
| /* divisor= */ MEDIA_CLOCK_FREQUENCY); | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please add comment on where these defaults are defined.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
RFC doesn't mandate any codec specific data(like width, height) to be present in fmtp attributes for VP8.
RFC also doesn't mentioned any default resolution. So, i use 320X240 as default since it is the default resolution used by Android Software decoder.
Adding the link for your reference:
https://cs.android.com/android/platform/superproject/+/master:frameworks/av/media/codec2/components/vpx/C2SoftVpxDec.cpp;drc=749a74cc3e081c16ea0e8c530953d0a247177867;l=70