[go: nahoru, domu]
Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rtp h265 #36

Merged
merged 6 commits into from
Feb 18, 2022
Merged

Rtp h265 #36

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4b99db9
Add support for RTSP H265
rakeshnitb Jan 13, 2022
9cb2436
Clean up RtpH265Reader
rakeshnitb Jan 28, 2022
ace363e
Fix review comments in RtpH265Reader
rakeshnitb Feb 8, 2022
3ef90d9
Fixed some of nitpick for RtpH265Reader
rakeshnitb Feb 9, 2022
aa68746
Add TODO for AggregationPacket mode
rakeshnitb Feb 11, 2022
ca1c1c2
Fix some minor review comments in RtpH265Reader
rakeshnitb Feb 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix review comments in RtpH265Reader
  • Loading branch information
@rakeshnitb
rakeshnitb committed Feb 8, 2022
commit ace363e1839c010755cefbf5e133367dd6da4370
56 changes: 29 additions & 27 deletions libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,26 @@ private static void processAacFmtpAttribute(
AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount)));
}

/** Returns H264/H265 initialization data from RTP parameter set. */
private static byte[] getInitializationDataFromParameterSet(String parameterSet) {
byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT);
byte[] decodedParameterNalUnit =
new byte[decodedParameterNalData.length + NAL_START_CODE.length];
System.arraycopy(
NAL_START_CODE,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ 0,
NAL_START_CODE.length);
System.arraycopy(
decodedParameterNalData,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ NAL_START_CODE.length,
decodedParameterNalData.length);
return decodedParameterNalUnit;
}

private static void processH264FmtpAttribute(
Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS));
Expand Down Expand Up @@ -200,25 +220,6 @@ private static void processH264FmtpAttribute(
}
}

private static byte[] getInitializationDataFromParameterSet(String parameterSet) {
byte[] decodedParameterNalData = Base64.decode(parameterSet, Base64.DEFAULT);
byte[] decodedParameterNalUnit =
new byte[decodedParameterNalData.length + NAL_START_CODE.length];
System.arraycopy(
NAL_START_CODE,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ 0,
NAL_START_CODE.length);
System.arraycopy(
decodedParameterNalData,
/* srcPos= */ 0,
decodedParameterNalUnit,
/* destPos= */ NAL_START_CODE.length,
decodedParameterNalData.length);
return decodedParameterNalUnit;
}

private static void processH265FmtpAttribute(
rakeshnitb marked this conversation as resolved.
Show resolved Hide resolved
Format.Builder formatBuilder, ImmutableMap<String, String> fmtpAttributes) {
if (fmtpAttributes.containsKey(PARAMETER_SPROP_H265_MAX_DON_DIFF)) {
Expand All @@ -228,23 +229,24 @@ private static void processH265FmtpAttribute(
"non-zero sprop-max-don-diff is not supported");
}

checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS));
String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS));
checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_SPS));
String spropSPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_SPS));
rakeshnitb marked this conversation as resolved.
Show resolved Hide resolved
checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_PPS));
String spropPPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_PPS));
checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_H265_VPS));
String spropVPS = checkNotNull(fmtpAttributes.get(PARAMETER_SPROP_H265_VPS));

byte[] vpsNalData = getInitializationDataFromParameterSet(spropVPS);
byte[] spsNalData = getInitializationDataFromParameterSet(spropSPS);
byte[] ppsNalData = getInitializationDataFromParameterSet(spropPPS);
ImmutableList<byte[]> initializationData = ImmutableList.of(vpsNalData, spsNalData, ppsNalData);
ImmutableList<byte[]> initializationData =
ImmutableList.of(
getInitializationDataFromParameterSet(spropVPS),
getInitializationDataFromParameterSet(spropSPS),
getInitializationDataFromParameterSet(spropPPS));
formatBuilder.setInitializationData(initializationData);

// Process SPS (Sequence Parameter Set).
byte[] spsNalDataWithStartCode = initializationData.get(1);
NalUnitUtil.H265SpsData spsData =
NalUnitUtil.parseH265SpsNalUnit(
spsNalData, NAL_START_CODE.length, spsNalData.length);
spsNalDataWithStartCode, NAL_START_CODE.length, spsNalDataWithStartCode.length);
formatBuilder.setPixelWidthHeightRatio(spsData.pixelWidthHeightRatio);
formatBuilder.setHeight(spsData.height);
formatBuilder.setWidth(spsData.width);
Expand Down
99 changes: 44 additions & 55 deletions ...ies/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpH265Reader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,17 +42,28 @@

private static final long MEDIA_CLOCK_FREQUENCY = 90_000;

/** Offset of payload data within a FU type A payload. */
/** Offset of payload data within a FU payload. */
private static final int FU_PAYLOAD_OFFSET = 3;

/** Single Time Aggregation Packet type A. */
private static final int RTP_PACKET_TYPE_STAP_A = 48; // RFC7798 Section 4.4.2
/** Fragmentation Unit type A. */
private static final int RTP_PACKET_TYPE_FU_A = 49;
/**
* Aggregation Packet.
*
* @see <a
* href="https://datatracker.ietf.org/doc/html/draft-ietf-payload-rtp-h265-15#section-4.4.2">
* RFC7798 Section 4.4.2</a>
*/
private static final int RTP_PACKET_TYPE_AP = 48;
/**
* Fragmentation Unit.
*
* @see <a
* href="https://datatracker.ietf.org/doc/html/draft-ietf-payload-rtp-h265-15#section-4.4.3">
* RFC7798 Section 4.4.3</a>
*/
private static final int RTP_PACKET_TYPE_FU = 49;

/** IDR NAL unit type. */
rakeshnitb marked this conversation as resolved.
Show resolved Hide resolved
private static final int NAL_IDR_W_LP = 19;

private static final int NAL_IDR_W_RADL = 19;
private static final int NAL_IDR_N_LP = 20;
rakeshnitb marked this conversation as resolved.
Show resolved Hide resolved

/** Scratch for Fragmentation Unit RTP packets. */
Expand All @@ -65,12 +76,10 @@

private @MonotonicNonNull TrackOutput trackOutput;
@C.BufferFlags private int bufferFlags;

private long firstReceivedTimestamp;
private int previousSequenceNumber;
/** The combined size of a sample that is fragmented into multiple RTP packets. */
private int fragmentedSampleSizeBytes;

private long startTimeOffsetUs;

/** Creates an instance. */
Expand All @@ -84,7 +93,6 @@ public RtpH265Reader(RtpPayloadFormat payloadFormat) {
@Override
public void createTracks(ExtractorOutput extractorOutput, int trackId) {
trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO);

castNonNull(trackOutput).format(payloadFormat.format);
}

Expand All @@ -94,7 +102,6 @@ public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {}
@Override
public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker)
throws ParserException {

int payloadType;
try {
// RFC7798 Section 1.1.4. NAL Unit Header
Expand All @@ -104,11 +111,11 @@ public void consume(ParsableByteArray data, long timestamp, int sequenceNumber,
}

checkStateNotNull(trackOutput);
if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_STAP_A) {
if (payloadType >= 0 && payloadType < RTP_PACKET_TYPE_AP) {
processSingleNalUnitPacket(data);
} else if (payloadType == RTP_PACKET_TYPE_STAP_A) {
processSingleTimeAggregationPacket(data);
} else if (payloadType == RTP_PACKET_TYPE_FU_A) {
} else if (payloadType == RTP_PACKET_TYPE_AP) {
processAggregationPacket(data);
} else if (payloadType == RTP_PACKET_TYPE_FU) {
processFragmentationUnitPacket(data, sequenceNumber);
} else {
throw ParserException.createForMalformedManifest(
Expand Down Expand Up @@ -173,46 +180,21 @@ private void processSingleNalUnitPacket(ParsableByteArray data) {
bufferFlags = getBufferFlagsFromNalType(nalHeaderType);
}


/**
* Processes STAP Type A packet (RFC7798 Section 4.4.2).
* Processes an AP packet (RFC7798 Section 4.4.2).
*
* <p>Outputs the received aggregation units (with start code prepended) to {@link #trackOutput}.
* Sets {@link #bufferFlags} and {@link #fragmentedSampleSizeBytes} accordingly.
*/
@RequiresNonNull("trackOutput")
private void processSingleTimeAggregationPacket(ParsableByteArray data) throws ParserException {
// An Example of an AP Packet Containing Two Aggregation
// Units without the DONL and DOND Fields.
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | RTP Header |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | PayloadHdr (Type=48) | NALU 1 Size |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | NALU 1 HDR | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- NALU 1 Data |
// | |
// | |
// + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | | NALU 2 Size | NALU 2 HDR |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | NALU 2 HDR | |
// +-+-+-+-+-+-+-+- NALU 2 Data |
// | |
// | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | :...OPTIONAL RTP padding |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

private void processAggregationPacket(ParsableByteArray data) throws ParserException {
throw ParserException.createForMalformedManifest(
"need to implement processSingleTimeAggregationPacket",
"need to implement processAggregationPacket",
/* cause= */ null);

}

/**
* Processes Fragmentation Unit Type A packet (RFC7798 Section 4.4.3).
* Processes Fragmentation Unit packet (RFC7798 Section 4.4.3).
*
* <p>This method will be invoked multiple times to receive a single frame that is broken down
* into a series of fragmentation units in multiple RTP packets.
Expand Down Expand Up @@ -241,15 +223,21 @@ private void processFragmentationUnitPacket(ParsableByteArray data, int packetSe
// +-+-+-+-+-+-+-+-+
// |S|E| FuType |
// +---------------+
rakeshnitb marked this conversation as resolved.
Show resolved Hide resolved

int tid = (data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header section 1.1.4
// Structure of HEVC NAL unit header
// +---------------+---------------+
// |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7|
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |F| Type | LayerId | TID |
// +-------------+-----------------+

int tid =
(data.getData()[1] & 0x7); // last 3 bits in byte 1 of payload header, RFC7798 Section 1.1.4
int fuHeader = data.getData()[2];
int nalUnitType = fuHeader & 0x3F;
byte nalHeader[] = new byte[2];
rakeshnitb marked this conversation as resolved.
Show resolved Hide resolved

nalHeader[0] = (byte) (nalUnitType << 1); // Section: 1.1.4
// According to section 1.1.4 in rfc7798, layerId is required to be zero so keeping its value
// zero and copying only tid.
nalHeader[0] = (byte) (nalUnitType << 1); // RFC7798 Section 1.1.4
// layerId must be zero according to RFC7798 Section 1.1.4, so copying the tid only
nalHeader[1] = (byte) tid;
boolean isFirstFuPacket = (fuHeader & 0x80) > 0;
boolean isLastFuPacket = (fuHeader & 0x40) > 0;
Expand All @@ -258,12 +246,13 @@ private void processFragmentationUnitPacket(ParsableByteArray data, int packetSe
// Prepends starter code.
fragmentedSampleSizeBytes += writeStartCode();

// The bytes needed is 2 (NALU header) + payload size. The original data array has size 3
// (2 payload + 1 FU header) + payload size. Thus setting the correct header and set position
// to 1.
// Overwrite byte 1 of payload header with byte 0 of HEVC nal header
// Overwrite a few bytes in Rtp buffer to get HEVC NAL unit
// Rtp Byte 0 -> Ignore
// Rtp Byte 1 -> Byte 0 of HEVC NAL header
// Rtp Byte 2 -> Byte 1 of HEVC NAL header
// Rtp Payload -> HEVC NAL bytes, so leave them unchanged
// Set data position from byte 1 as byte 0 was ignored
data.getData()[1] = (byte) nalHeader[0];
// Overwrite byte FU Header with byte 1 of HEVC nal header
data.getData()[2] = (byte) nalHeader[1];
fuScratchBuffer.reset(data.getData());
fuScratchBuffer.setPosition(1);
Expand Down Expand Up @@ -312,6 +301,6 @@ private static long toSampleUs(

@C.BufferFlags
private static int getBufferFlagsFromNalType(int nalType) {
return (nalType == NAL_IDR_W_LP || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0;
return (nalType == NAL_IDR_W_RADL || nalType == NAL_IDR_N_LP) ? C.BUFFER_FLAG_KEY_FRAME : 0;
}
}