From 838e21f6e7f8b21d1b74422d5eaa0fd236331858 Mon Sep 17 00:00:00 2001 From: Sean DuBois Date: Fri, 29 Dec 2023 22:34:32 -0500 Subject: [PATCH] Add H264RtpDepacketizer Inverse of H264RtpPacketizer. Takes incoming H264 packets and emits H264 NALUs. Co-authored-by: Paul-Louis Ageneau --- CMakeLists.txt | 2 + include/rtc/h264rtpdepacketizer.hpp | 42 +++++++++ include/rtc/rtc.hpp | 1 + src/h264rtpdepacketizer.cpp | 141 ++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 include/rtc/h264rtpdepacketizer.hpp create mode 100644 src/h264rtpdepacketizer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 87d309c2c..05993588a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,7 @@ set(LIBDATACHANNEL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/rtcpsrreporter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/rtppacketizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/h264rtppacketizer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/h264rtpdepacketizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/nalunit.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/h265rtppacketizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/h265nalunit.cpp @@ -110,6 +111,7 @@ set(LIBDATACHANNEL_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/rtcpsrreporter.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/rtppacketizer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264rtppacketizer.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h264rtpdepacketizer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/nalunit.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265rtppacketizer.hpp ${CMAKE_CURRENT_SOURCE_DIR}/include/rtc/h265nalunit.hpp diff --git a/include/rtc/h264rtpdepacketizer.hpp b/include/rtc/h264rtpdepacketizer.hpp new file mode 100644 index 000000000..663e459a4 --- /dev/null +++ b/include/rtc/h264rtpdepacketizer.hpp @@ -0,0 +1,42 @@ +/** + * Copyright (c) 2020 Staz Modrzynski + * Copyright (c) 2020 Paul-Louis Ageneau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#ifndef RTC_H264_RTP_DEPACKETIZER_H +#define RTC_H264_RTP_DEPACKETIZER_H + +#if RTC_ENABLE_MEDIA + +#include "common.hpp" +#include "mediahandler.hpp" +#include "message.hpp" +#include "rtp.hpp" + +#include + +namespace rtc { + +/// RTP depacketization for H264 +class RTC_CPP_EXPORT H264RtpDepacketizer : public MediaHandler { +public: + H264RtpDepacketizer() = default; + virtual ~H264RtpDepacketizer() = default; + + void incoming(message_vector &messages, const message_callback &send) override; + +private: + std::vector mRtpBuffer; + + message_vector buildFrames(message_vector::iterator firstPkt, message_vector::iterator lastPkt); +}; + +} // namespace rtc + +#endif // RTC_ENABLE_MEDIA + +#endif /* RTC_H264_RTP_DEPACKETIZER_H */ diff --git a/include/rtc/rtc.hpp b/include/rtc/rtc.hpp index ed10fa31f..dc7f62eac 100644 --- a/include/rtc/rtc.hpp +++ b/include/rtc/rtc.hpp @@ -30,6 +30,7 @@ // Media #include "av1rtppacketizer.hpp" #include "h264rtppacketizer.hpp" +#include "h264rtpdepacketizer.hpp" #include "h265rtppacketizer.hpp" #include "mediahandler.hpp" #include "plihandler.hpp" diff --git a/src/h264rtpdepacketizer.cpp b/src/h264rtpdepacketizer.cpp new file mode 100644 index 000000000..ba38aab1d --- /dev/null +++ b/src/h264rtpdepacketizer.cpp @@ -0,0 +1,141 @@ +/** + * Copyright (c) 2023 Paul-Louis Ageneau + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +#if RTC_ENABLE_MEDIA + +#include "h264rtpdepacketizer.hpp" +#include "track.hpp" + +#include "impl/logcounter.hpp" + +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace rtc { + +const unsigned long stapaHeaderSize = 1; +const auto fuaHeaderSize = 2; + +const uint8_t naluTypeBitmask = 0x1F; +const uint8_t naluTypeSTAPA = 24; +const uint8_t naluTypeFUA = 28; +const uint8_t fuaEndBitmask = 0x40; +const uint8_t naluRefIdcBitmask = 0x60; + +message_vector H264RtpDepacketizer::buildFrames(message_vector::iterator begin, + message_vector::iterator end) { + message_vector out = {}; + auto fua_buffer = std::vector{}; + + for (auto it = begin; it != end; it++) { + auto pkt = it->get(); + auto pktParsed = reinterpret_cast(pkt->data()); + auto headerSize = + sizeof(rtc::RtpHeader) + pktParsed->csrcCount() + pktParsed->getExtensionHeaderSize(); + auto firstByte = std::to_integer(pkt->at(headerSize)); + auto secondByte = std::to_integer(pkt->at(headerSize + 1)); + auto naluType = firstByte & naluTypeBitmask; + + if (fua_buffer.size() != 0 || naluType == naluTypeFUA) { + if (fua_buffer.size() == 0) { + fua_buffer.push_back(std::byte(0)); + } + + std::copy(pkt->begin() + headerSize + fuaHeaderSize, pkt->end(), + std::back_inserter(fua_buffer)); + + if ((secondByte & fuaEndBitmask) != 0) { + auto naluRefIdc = firstByte & naluRefIdcBitmask; + auto fragmentedNaluType = secondByte & naluTypeBitmask; + fua_buffer.at(0) = std::byte(naluRefIdc | fragmentedNaluType); + + out.push_back(make_message(std::move(fua_buffer))); + fua_buffer.clear(); + } + } else if (naluType > 0 && naluType < 24) { + out.push_back(make_message(pkt->begin() + headerSize, pkt->end())); + } else if (naluType == naluTypeSTAPA) { + auto currOffset = stapaHeaderSize + headerSize; + + while (currOffset < pkt->size()) { + auto naluSize = + uint16_t(pkt->at(currOffset)) << 8 | uint8_t(pkt->at(currOffset + 1)); + + currOffset += 2; + + if (pkt->size() < currOffset + naluSize) { + throw std::runtime_error("STAP-A declared size is larger then buffer"); + } + + out.push_back( + make_message(pkt->begin() + currOffset, pkt->begin() + currOffset + naluSize)); + currOffset += naluSize; + } + + } else { + throw std::runtime_error("Unknown H264 RTP Packetization"); + } + } + + return out; +} + +void H264RtpDepacketizer::incoming(message_vector &messages, const message_callback &) { + for (auto message : messages) { + if (message->type == Message::Control) { + continue; // RTCP + } + + if (message->size() < sizeof(RtpHeader)) { + PLOG_VERBOSE << "RTP packet is too small, size=" << message->size(); + continue; + } + + mRtpBuffer.push_back(message); + } + + messages.clear(); + + while (mRtpBuffer.size() != 0) { + uint32_t current_timestamp = 0; + size_t packets_in_timestamp = 0; + + for (const auto &pkt : mRtpBuffer) { + auto p = reinterpret_cast(pkt->data()); + + if (current_timestamp == 0) { + current_timestamp = p->timestamp(); + } else if (current_timestamp != p->timestamp()) { + break; + } + + packets_in_timestamp++; + } + + if (packets_in_timestamp == mRtpBuffer.size()) { + break; + } + + auto begin = mRtpBuffer.begin(); + auto end = mRtpBuffer.begin() + (packets_in_timestamp - 1); + + auto frames = buildFrames(begin, end + 1); + messages.insert(messages.end(), frames.begin(), frames.end()); + mRtpBuffer.erase(mRtpBuffer.begin(), mRtpBuffer.begin() + packets_in_timestamp); + } +} + +} // namespace rtc + +#endif // RTC_ENABLE_MEDIA