From c4978314128eafe03623d4800e876eeaef21c338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Mon, 9 Feb 2026 18:27:57 +0100 Subject: [PATCH 01/10] Worker `MultiStreamConsumer` # Details - `MultiStreamConsumer` replaces `SimpleConsumer` and `SimulcastConsumer` and merges both. ## TODO I've renamed `TestSimpleConsumer.cpp` to `TestMultiStreamConsumer.cpp` and it fails miserably. And I don't understand why. Run it as follows: ```bash MEDIASOUP_TEST_TAGS="[consumer]" MS_TEST_LOG_LEVEL=debug MS_TEST_LOG_TAGS="rtp simulcast" make test ``` Output: ``` RTC::MultiStreamConsumer::MultiStreamConsumer() | ------ encoding.ssrc:1234567890 RTC::MultiStreamConsumer::CreateRtpStream() | [ssrc:1234567890, payloadType:111] RTC::MultiStreamConsumer::SendRtpPacket() | ---- 5 packet->GetSsrc():5, mapMappedSsrcSpatialLayer.size():1 RTC::MultiStreamConsumer::SendRtpPacket() | ------ 5.1 mapMappedSsrcSpatialLayer> RTC::MultiStreamConsumer::SendRtpPacket() | ------------ mappeSsrc:1234567890, spatialLayer:0 RTC::MultiStreamConsumer::SendRtpPacket() | ------ 5.1 mapMappedSsrcSpatialLayer> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ mediasoup-worker-test is a Catch2 v3.12.0 host application. Run with -? for options ------------------------------------------------------------------------------- Scenario: MultiStreamConsumer RTP packets are not forwarded when the consumer is not active ------------------------------------------------------------------------------- ../../../test/src/RTC/TestMultiStreamConsumer.cpp:232 ............................................................................... ../../../test/src/RTC/TestMultiStreamConsumer.cpp:232: FAILED: {Unknown expression after the reported line} due to unexpected exception with message: absl::container_internal::raw_hash_map<>::at ``` Pay attention to these temporal logs I added in `SendRtpPacket()`: ```c++ void MultiStreamConsumer::SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) { MS_TRACE(); #ifdef MS_RTC_LOGGER_RTP packet->logger.consumerId = this->id; #endif // TODO MS_DUMP( "---- 5 packet->GetSsrc():%" PRIu32 ", mapMappedSsrcSpatialLayer.size():%zu", packet->GetSsrc(), this->mapMappedSsrcSpatialLayer.size()); // TODO MS_DUMP("------ 5.1 mapMappedSsrcSpatialLayer>"); for (const auto& kv : this->mapMappedSsrcSpatialLayer) { MS_DUMP("------------ mappeSsrc:%" PRIu32 ", spatialLayer:%" PRIi16, kv.first, kv.second); } MS_DUMP("------ 5.1 mapMappedSsrcSpatialLayer>"); auto spatialLayer = this->mapMappedSsrcSpatialLayer.at(packet->GetSsrc()); // TODO MS_DUMP("----- 5.2 OK!!!"); ``` How the hell is possible that it crashes in `auto spatialLayer = this->mapMappedSsrcSpatialLayer.at(packet->GetSsrc())` line if it's clear that an entry with key (`mappedSsrc: 1234567890`) is present in `this->mapMappedSsrcSpatialLayer`?. --- worker/include/RTC/Consumer.hpp | 1 + worker/include/RTC/MultiStreamConsumer.hpp | 144 ++ worker/include/RTC/SimpleConsumer.hpp | 1 - worker/include/RTC/SimulcastConsumer.hpp | 1 - worker/meson.build | 5 +- worker/src/RTC/MultiStreamConsumer.cpp | 2008 +++++++++++++++++ worker/src/RTC/SimpleConsumer.cpp | 3 +- worker/src/RTC/Transport.cpp | 9 +- ...nsumer.cpp => TestMultiStreamConsumer.cpp} | 112 +- 9 files changed, 2217 insertions(+), 67 deletions(-) create mode 100644 worker/include/RTC/MultiStreamConsumer.hpp create mode 100644 worker/src/RTC/MultiStreamConsumer.cpp rename worker/test/src/RTC/{TestSimpleConsumer.cpp => TestMultiStreamConsumer.cpp} (73%) diff --git a/worker/include/RTC/Consumer.hpp b/worker/include/RTC/Consumer.hpp index faf9526fec..86f6333342 100644 --- a/worker/include/RTC/Consumer.hpp +++ b/worker/include/RTC/Consumer.hpp @@ -5,6 +5,7 @@ #include "Channel/ChannelRequest.hpp" #include "Channel/ChannelSocket.hpp" #include "FBS/consumer.h" +#include "FBS/transport.h" #include "RTC/ConsumerTypes.hpp" #include "RTC/RTCP/CompoundPacket.hpp" #include "RTC/RTCP/FeedbackRtpNack.hpp" diff --git a/worker/include/RTC/MultiStreamConsumer.hpp b/worker/include/RTC/MultiStreamConsumer.hpp new file mode 100644 index 0000000000..9d91d2c15f --- /dev/null +++ b/worker/include/RTC/MultiStreamConsumer.hpp @@ -0,0 +1,144 @@ +#ifndef MS_RTC_MULTI_STREAM_CONSUMER_HPP +#define MS_RTC_MULTI_STREAM_CONSUMER_HPP + +#include "FBS/consumer.h" +#include "RTC/Consumer.hpp" +#include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" +#include "RTC/SeqManager.hpp" +#include "RTC/Shared.hpp" +#include + +namespace RTC +{ + class MultiStreamConsumer : public RTC::Consumer, public RTC::RTP::RtpStreamSend::Listener + { + public: + MultiStreamConsumer( + RTC::Shared* shared, + const std::string& id, + const std::string& producerId, + RTC::RtpParameters::Type type, + RTC::Consumer::Listener* listener, + const FBS::Transport::ConsumeRequest* data); + ~MultiStreamConsumer() override; + + public: + flatbuffers::Offset FillBuffer( + flatbuffers::FlatBufferBuilder& builder) const; + flatbuffers::Offset FillBufferStats( + flatbuffers::FlatBufferBuilder& builder) override; + flatbuffers::Offset FillBufferScore( + flatbuffers::FlatBufferBuilder& builder) const override; + RTC::ConsumerTypes::VideoLayers GetPreferredLayers() const override + { + RTC::ConsumerTypes::VideoLayers layers; + + layers.spatial = this->preferredLayers.spatial; + layers.temporal = this->preferredLayers.temporal; + + return layers; + } + bool IsActive() const override + { + // clang-format off + return ( + RTC::Consumer::IsActive() && + std::any_of( + this->producerRtpStreams.begin(), + this->producerRtpStreams.end(), + [](const RTC::RTP::RtpStreamRecv* rtpStream) + { + // If there is no RTP inactivity check do not consider the stream + // inactive despite it has score 0. + return (rtpStream != nullptr && (rtpStream->GetScore() > 0u || !rtpStream->HasRtpInactivityCheckEnabled())); + } + ) + ); + // clang-format on + } + void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; + void ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; + void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; + uint8_t GetBitratePriority() const override; + uint32_t IncreaseLayer(uint32_t bitrate, bool considerLoss) override; + void ApplyLayers() override; + uint32_t GetDesiredBitrate() const override; + void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) override; + bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) override; + const std::vector& GetRtpStreams() const override + { + return this->rtpStreams; + } + void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override; + void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) override; + void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) override; + void ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) override; + void ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) override; + uint32_t GetTransmissionRate(uint64_t nowMs) override; + float GetRtt() const override; + + /* Methods inherited from Channel::ChannelSocket::RequestHandler. */ + public: + void HandleRequest(Channel::ChannelRequest* request) override; + + private: + void UserOnTransportConnected() override; + void UserOnTransportDisconnected() override; + void UserOnPaused() override; + void UserOnResumed() override; + void CreateRtpStream(); + void RequestKeyFrames(); + void RequestKeyFrameForTargetSpatialLayer(); + void RequestKeyFrameForCurrentSpatialLayer(); + void MayChangeLayers(bool force = false); + bool RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const; + void UpdateTargetLayers(int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer); + bool CanSwitchToSpatialLayer(int16_t spatialLayer) const; + void EmitScore() const; + void StorePacketInTargetLayerRetransmissionBuffer( + RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket); + void EmitLayersChange() const; + RTC::RTP::RtpStreamRecv* GetProducerCurrentRtpStream() const; + RTC::RTP::RtpStreamRecv* GetProducerTargetRtpStream() const; + RTC::RTP::RtpStreamRecv* GetProducerTsReferenceRtpStream() const; + + /* Pure virtual methods inherited from RtpStreamSend::Listener. */ + public: + void OnRtpStreamScore(RTC::RTP::RtpStream* rtpStream, uint8_t score, uint8_t previousScore) override; + void OnRtpStreamRetransmitRtpPacket( + RTC::RTP::RtpStreamSend* rtpStream, RTC::RTP::Packet* packet) override; + + private: + // Allocated by this. + RTC::RTP::RtpStreamSend* rtpStream{ nullptr }; + // Others. + absl::flat_hash_map mapMappedSsrcSpatialLayer; + std::vector rtpStreams; + std::vector producerRtpStreams; // Indexed by spatial layer. + bool keyFrameSupported{ false }; + bool syncRequired{ false }; + int16_t spatialLayerToSync{ -1 }; + bool lastSentPacketHasMarker{ false }; + RTC::SeqManager rtpSeqManager; + RTC::ConsumerTypes::VideoLayers preferredLayers; + RTC::ConsumerTypes::VideoLayers provisionalTargetLayers; + RTC::ConsumerTypes::VideoLayers targetLayers; + int16_t currentSpatialLayer{ -1 }; + int16_t tsReferenceSpatialLayer{ -1 }; // Used for RTP TS sync. + uint16_t snReferenceSpatialLayer{ 0 }; + bool checkingForOldPacketsInSpatialLayer{ false }; + std::unique_ptr encodingContext; + uint32_t tsOffset{ 0u }; // RTP Timestamp offset. + bool keyFrameForTsOffsetRequested{ false }; + // Last time we moved to lower spatial layer due to BWE. + uint64_t lastBweDowngradeAtMs{ 0u }; + // Buffer to store packets that arrive earlier than the first packet of the + // video key frame. + std::map::SeqLowerThan> + targetLayerRetransmissionBuffer; + }; +} // namespace RTC + +#endif diff --git a/worker/include/RTC/SimpleConsumer.hpp b/worker/include/RTC/SimpleConsumer.hpp index 83b8f9c735..0e69241573 100644 --- a/worker/include/RTC/SimpleConsumer.hpp +++ b/worker/include/RTC/SimpleConsumer.hpp @@ -1,7 +1,6 @@ #ifndef MS_RTC_SIMPLE_CONSUMER_HPP #define MS_RTC_SIMPLE_CONSUMER_HPP -#include "FBS/transport.h" #include "RTC/Consumer.hpp" #include "RTC/SeqManager.hpp" #include "RTC/Shared.hpp" diff --git a/worker/include/RTC/SimulcastConsumer.hpp b/worker/include/RTC/SimulcastConsumer.hpp index c63d7c8254..6654be5a49 100644 --- a/worker/include/RTC/SimulcastConsumer.hpp +++ b/worker/include/RTC/SimulcastConsumer.hpp @@ -1,7 +1,6 @@ #ifndef MS_RTC_SIMULCAST_CONSUMER_HPP #define MS_RTC_SIMULCAST_CONSUMER_HPP -#include "FBS/consumer.h" #include "RTC/Consumer.hpp" #include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" #include "RTC/SeqManager.hpp" diff --git a/worker/meson.build b/worker/meson.build index cb48c93bf3..ec1d3fdd52 100644 --- a/worker/meson.build +++ b/worker/meson.build @@ -126,8 +126,7 @@ common_sources = [ 'src/RTC/SeqManager.cpp', 'src/RTC/Serializable.cpp', 'src/RTC/Shared.cpp', - 'src/RTC/SimpleConsumer.cpp', - 'src/RTC/SimulcastConsumer.cpp', + 'src/RTC/MultiStreamConsumer.cpp', 'src/RTC/SrtpSession.cpp', 'src/RTC/SvcConsumer.cpp', 'src/RTC/TcpConnection.cpp', @@ -410,7 +409,7 @@ test_sources = [ 'test/src/RTC/TestRateCalculator.cpp', 'test/src/RTC/TestRtpEncodingParameters.cpp', 'test/src/RTC/TestSeqManager.cpp', - 'test/src/RTC/TestSimpleConsumer.cpp', + 'test/src/RTC/TestMultiStreamConsumer.cpp', 'test/src/RTC/TestTransportCongestionControlServer.cpp', 'test/src/RTC/TestTransportTuple.cpp', 'test/src/RTC/TestTrendCalculator.cpp', diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp new file mode 100644 index 0000000000..a84190457e --- /dev/null +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -0,0 +1,2008 @@ +#define MS_CLASS "RTC::MultiStreamConsumer" +// TODO: COmment +#define MS_LOG_DEV_LEVEL 3 + +#include "RTC/MultiStreamConsumer.hpp" +#include "DepLibUV.hpp" +#include "Logger.hpp" +#include "MediaSoupErrors.hpp" +#include "Utils.hpp" +#include "RTC/RTP/Codecs/Tools.hpp" +#ifdef MS_RTC_LOGGER_RTP +#include "RTC/RtcLogger.hpp" +#endif +#include // std::numeric_limits + +namespace RTC +{ + /* Static. */ + + static constexpr uint64_t StreamMinActiveMs{ 2000u }; + static constexpr uint64_t BweDowngradeConservativeMs{ 10000u }; + static constexpr uint64_t BweDowngradeMinActiveMs{ 8000u }; + static constexpr uint16_t MaxSequenceNumberGap{ 100u }; + static constexpr size_t TargetLayerRetransmissionBufferSize{ 30u }; + + /* Instance methods. */ + + MultiStreamConsumer::MultiStreamConsumer( + RTC::Shared* shared, + const std::string& id, + const std::string& producerId, + RTC::RtpParameters::Type type, + RTC::Consumer::Listener* listener, + const FBS::Transport::ConsumeRequest* data) + : RTC::Consumer::Consumer(shared, id, producerId, listener, data, type) + { + MS_TRACE(); + + MS_ASSERT( + this->type == RTC::RtpParameters::Type::SIMPLE || + this->type == RTC::RtpParameters::Type::SIMULCAST, + "type must be SIMPLE or SIMULCAST"); + + // We allow a single encoding in simulcast (so we can enable temporal layers + // with a single simulcast stream). + // NOTE: No need to check this->consumableRtpEncodings.size() > 0 here since + // it's already done in Consumer constructor. + + auto& encoding = this->rtpParameters.encodings[0]; + const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); + + // Ensure there are as many spatial layers as encodings. + if (encoding.spatialLayers != this->consumableRtpEncodings.size()) + { + MS_THROW_TYPE_ERROR("encoding.spatialLayers does not match number of consumableRtpEncodings"); + } + + if (!RTC::RTP::Codecs::Tools::IsValidTypeForCodec(this->type, mediaCodec->mimeType)) + { + MS_THROW_TYPE_ERROR( + "%s codec not supported for simulcast", mediaCodec->mimeType.ToString().c_str()); + } + + // Fill mapMappedSsrcSpatialLayer. + for (size_t idx{ 0u }; idx < this->consumableRtpEncodings.size(); ++idx) + { + auto& encoding = this->consumableRtpEncodings[idx]; + + // TODO + MS_DUMP("------ encoding.ssrc:%" PRIu32, encoding.ssrc); + + this->mapMappedSsrcSpatialLayer[encoding.ssrc] = static_cast(idx); + } + + this->keyFrameSupported = RTC::RTP::Codecs::Tools::CanBeKeyFrame(mediaCodec->mimeType); + + // Set preferredLayers (if given). + if (flatbuffers::IsFieldPresent(data, FBS::Transport::ConsumeRequest::VT_PREFERREDLAYERS)) + { + const auto* preferredLayers = data->preferredLayers(); + + this->preferredLayers.spatial = preferredLayers->spatialLayer(); + + if (this->preferredLayers.spatial > encoding.spatialLayers - 1) + { + this->preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); + } + + if (auto preferredTemporalLayer = preferredLayers->temporalLayer(); + preferredTemporalLayer.has_value()) + { + this->preferredLayers.temporal = preferredTemporalLayer.value(); + + if (this->preferredLayers.temporal > encoding.temporalLayers - 1) + { + this->preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + } + else + { + this->preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + } + else + { + // Initially set preferredSpatialLayer and preferredTemporalLayer to the + // maximum value. + this->preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); + this->preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); + } + + // Reserve space for the Producer RTP streams by filling all the possible + // entries with nullptr. + this->producerRtpStreams.insert( + this->producerRtpStreams.begin(), this->consumableRtpEncodings.size(), nullptr); + + // Let's chosee an initial output seq number between 1000 and 32768 to avoid + // libsrtp bug: + // https://github.com/versatica/mediasoup/issues/1437 + const uint16_t initialOutputSeq = + Utils::Crypto::GetRandomUInt(1000u, std::numeric_limits::max() / 2); + + this->rtpSeqManager = RTC::SeqManager(initialOutputSeq); + + // Create the encoding context. + RTC::RTP::Codecs::EncodingContext::Params params; + + params.spatialLayers = encoding.spatialLayers; + params.temporalLayers = encoding.temporalLayers; + + this->encodingContext.reset( + RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params)); + + // Specific for Opus codec. + if ( + this->encodingContext && mediaCodec->mimeType.type == RTC::RtpCodecMimeType::Type::AUDIO && + (mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::OPUS || + mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::MULTIOPUS)) + { + // ignoreDtx is set to false by default. + this->encodingContext->SetIgnoreDtx(data->ignoreDtx()); + } + + // Create RtpStreamSend instance for sending a single stream to the remote. + CreateRtpStream(); + + // NOTE: This may throw. + this->shared->channelMessageRegistrator->RegisterHandler( + this->id, + /*channelRequestHandler*/ this, + /*channelRequestHandler*/ nullptr); + } + + MultiStreamConsumer::~MultiStreamConsumer() + { + MS_TRACE(); + + this->shared->channelMessageRegistrator->UnregisterHandler(this->id); + + delete this->rtpStream; + this->targetLayerRetransmissionBuffer.clear(); + } + + flatbuffers::Offset MultiStreamConsumer::FillBuffer( + flatbuffers::FlatBufferBuilder& builder) const + { + MS_TRACE(); + + // Call the parent method. + auto base = RTC::Consumer::FillBuffer(builder); + // Add rtpStream. + std::vector> rtpStreams; + rtpStreams.emplace_back(this->rtpStream->FillBuffer(builder)); + + auto dump = FBS::Consumer::CreateConsumerDumpDirect( + builder, + base, + &rtpStreams, + this->preferredLayers.spatial, + this->targetLayers.spatial, + this->currentSpatialLayer, + this->preferredLayers.temporal, + this->targetLayers.temporal, + this->encodingContext ? this->encodingContext->GetCurrentTemporalLayer() : 1); + + return FBS::Consumer::CreateDumpResponse(builder, dump); + } + + flatbuffers::Offset MultiStreamConsumer::FillBufferStats( + flatbuffers::FlatBufferBuilder& builder) + { + MS_TRACE(); + + std::vector> rtpStreams; + + // Add stats of our send stream. + rtpStreams.emplace_back(this->rtpStream->FillBufferStats(builder)); + + auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); + + // Add stats of our recv stream. + if (producerCurrentRtpStream) + { + rtpStreams.emplace_back(producerCurrentRtpStream->FillBufferStats(builder)); + } + + return FBS::Consumer::CreateGetStatsResponseDirect(builder, &rtpStreams); + } + + flatbuffers::Offset MultiStreamConsumer::FillBufferScore( + flatbuffers::FlatBufferBuilder& builder) const + { + MS_TRACE(); + + MS_ASSERT(this->producerRtpStreamScores, "producerRtpStreamScores not set"); + + auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); + + uint8_t producerScore{ 0 }; + + if (producerCurrentRtpStream) + { + producerScore = producerCurrentRtpStream->GetScore(); + } + else + { + producerScore = 0; + } + + return FBS::Consumer::CreateConsumerScoreDirect( + builder, this->rtpStream->GetScore(), producerScore, this->producerRtpStreamScores); + } + + void MultiStreamConsumer::HandleRequest(Channel::ChannelRequest* request) + { + MS_TRACE(); + + switch (request->method) + { + case Channel::ChannelRequest::Method::CONSUMER_DUMP: + { + auto dumpOffset = FillBuffer(request->GetBufferBuilder()); + + request->Accept(FBS::Response::Body::Consumer_DumpResponse, dumpOffset); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_REQUEST_KEY_FRAME: + { + if (IsActive()) + { + RequestKeyFrames(); + } + + request->Accept(); + + break; + } + + case Channel::ChannelRequest::Method::CONSUMER_SET_PREFERRED_LAYERS: + { + auto previousPreferredLayers = this->preferredLayers; + + const auto* body = request->data->body_as(); + const auto* preferredLayers = body->preferredLayers(); + + // Spatial layer. + this->preferredLayers.spatial = preferredLayers->spatialLayer(); + + if (this->preferredLayers.spatial > this->rtpStream->GetSpatialLayers() - 1) + { + this->preferredLayers.spatial = + static_cast(this->rtpStream->GetSpatialLayers() - 1); + } + + // preferredTemporaLayer is optional. + auto preferredTemporalLayer = preferredLayers->temporalLayer(); + + if (preferredTemporalLayer.has_value()) + { + this->preferredLayers.temporal = preferredTemporalLayer.value(); + + if (this->preferredLayers.temporal > this->rtpStream->GetTemporalLayers() - 1) + { + this->preferredLayers.temporal = + static_cast(this->rtpStream->GetTemporalLayers() - 1); + } + } + else + { + this->preferredLayers.temporal = + static_cast(this->rtpStream->GetTemporalLayers() - 1); + } + + MS_DEBUG_DEV( + "preferred layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", + this->preferredLayers.spatial, + this->preferredLayers.temporal, + this->id.c_str()); + + preferredTemporalLayer = this->preferredLayers.temporal; + auto preferredLayersOffset = FBS::Consumer::CreateConsumerLayers( + request->GetBufferBuilder(), this->preferredLayers.spatial, preferredTemporalLayer); + auto responseOffset = FBS::Consumer::CreateSetPreferredLayersResponse( + request->GetBufferBuilder(), preferredLayersOffset); + + request->Accept(FBS::Response::Body::Consumer_SetPreferredLayersResponse, responseOffset); + + if (IsActive() && this->preferredLayers != previousPreferredLayers) + { + MayChangeLayers(/*force*/ true); + } + + break; + } + + default: + { + // Pass it to the parent class. + RTC::Consumer::HandleRequest(request); + } + } + } + + void MultiStreamConsumer::ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) + { + MS_TRACE(); + + // TODO + MS_DUMP("------ mappedSsrc:%" PRIu32, mappedSsrc); + + auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); + + MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); + + const int16_t spatialLayer = it->second; + + this->producerRtpStreams[spatialLayer] = rtpStream; + } + + void MultiStreamConsumer::ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) + { + MS_TRACE(); + + // TODO + MS_DUMP("------ mappedSsrc:%" PRIu32, mappedSsrc); + + auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); + + MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); + + const int16_t spatialLayer = it->second; + + this->producerRtpStreams[spatialLayer] = rtpStream; + + // Emit the score event. + EmitScore(); + + if (IsActive()) + { + MayChangeLayers(); + } + } + + void MultiStreamConsumer::ProducerRtpStreamScore( + RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t score, uint8_t previousScore) + { + MS_TRACE(); + + // Emit the score event. + EmitScore(); + + if (RTC::Consumer::IsActive()) + { + // All Producer streams are dead. + if (!IsActive()) + { + UpdateTargetLayers(-1, -1); + } + // Just check target layers if the stream has died or reborned. + else if (!this->externallyManagedBitrate || (score == 0u || previousScore == 0u)) + { + MayChangeLayers(); + } + } + } + + void MultiStreamConsumer::ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) + { + MS_TRACE(); + + // Just interested if this is the first Sender Report for a RTP stream. + if (!first) + { + return; + } + + MS_DEBUG_TAG(simulcast, "first SenderReport [ssrc:%" PRIu32 "]", rtpStream->GetSsrc()); + + // If our RTP timestamp reference stream does not yet have SR, do nothing + // since we know we won't be able to switch. + auto* producerTsReferenceRtpStream = GetProducerTsReferenceRtpStream(); + + if (!producerTsReferenceRtpStream || !producerTsReferenceRtpStream->GetSenderReportNtpMs()) + { + return; + } + + if (IsActive()) + { + MayChangeLayers(); + } + } + + uint8_t MultiStreamConsumer::GetBitratePriority() const + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + + // Audio SimpleConsumer does not play the BWE game. + if (this->kind != RTC::Media::Kind::VIDEO) + { + return 0u; + } + + if (!IsActive()) + { + return 0u; + } + + return this->priority; + } + + uint32_t MultiStreamConsumer::IncreaseLayer(uint32_t bitrate, bool considerLoss) + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + MS_ASSERT(this->kind == RTC::Media::Kind::VIDEO, "should be video"); + MS_ASSERT(IsActive(), "should be active"); + + // If already in the preferred layers, do nothing. + if (this->provisionalTargetLayers == this->preferredLayers) + { + return 0u; + } + + uint32_t virtualBitrate; + + if (considerLoss) + { + // Calculate virtual available bitrate based on given bitrate and our + // packet lost. + auto lossPercentage = this->rtpStream->GetLossPercentage(); + + if (lossPercentage < 2) + { + virtualBitrate = 1.08 * bitrate; + } + else if (lossPercentage > 10) + { + virtualBitrate = (1 - 0.5 * (lossPercentage / 100)) * bitrate; + } + else + { + virtualBitrate = bitrate; + } + } + else + { + virtualBitrate = bitrate; + } + + uint32_t requiredBitrate{ 0u }; + int16_t spatialLayer{ 0 }; + int16_t temporalLayer{ 0 }; + auto nowMs = DepLibUV::GetTimeMs(); + + for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx) + { + spatialLayer = static_cast(sIdx); + + // If this is higher than current spatial layer and we moved to to current + // spatial layer due to BWE limitations, check how much it has elapsed + // since then. + if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) + { + if (this->provisionalTargetLayers.spatial > -1 && spatialLayer > this->currentSpatialLayer) + { + MS_DEBUG_DEV( + "avoid upgrading to spatial layer %" PRIi16 " due to recent BWE downgrade", spatialLayer); + + goto done; + } + } + + // Ignore spatial layers lower than the one we already have. + if (spatialLayer < this->provisionalTargetLayers.spatial) + { + continue; + } + // If this is the higher than preferred spatial layer, abort. + else if (spatialLayer > this->preferredLayers.spatial) + { + MS_DEBUG_DEV( + "avoid upgrading to spatial layer %" PRIi16 + " since it's higher than preferred spatial layer %" PRIi16, + spatialLayer, + this->preferredLayers.spatial); + + goto done; + } + + // TODO + MS_DUMP( + "---- 1 spatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", + spatialLayer, + this->producerRtpStreams.size()); + + // This can be null. + auto* producerRtpStream = this->producerRtpStreams.at(spatialLayer); + + // Producer stream does not exist. Ignore. + if (!producerRtpStream) + { + continue; + } + + // Ignore spatial layers (streams) with score 0. + if (producerRtpStream->GetScore() == 0) + { + continue; + } + + // If the stream has not been active time enough and we have an active one + // already, move to the next spatial layer. + if ( + spatialLayer != this->provisionalTargetLayers.spatial && + this->provisionalTargetLayers.spatial != -1 && + producerRtpStream->GetActiveMs() < StreamMinActiveMs) + { + // TODO + MS_DUMP( + "---- 2 this->provisionalTargetLayers.spatial:%" PRIi16 ", producerRtpStreams.size():%zu", + this->provisionalTargetLayers.spatial, + this->producerRtpStreams.size()); + + const auto* provisionalProducerRtpStream = + this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); + + // The stream for the current provisional spatial layer has been active + // for enough time, move to the next spatial layer. + if (provisionalProducerRtpStream->GetActiveMs() >= StreamMinActiveMs) + { + continue; + } + } + + // We may not yet switch to this spatial layer. + if (!CanSwitchToSpatialLayer(spatialLayer)) + { + continue; + } + + temporalLayer = 0; + + // Check bitrate of every temporal layer. + for (; temporalLayer < producerRtpStream->GetTemporalLayers(); ++temporalLayer) + { + // Ignore temporal layers lower than the one we already have (taking + // into account the spatial layer too). + if ( + spatialLayer == this->provisionalTargetLayers.spatial && + temporalLayer <= this->provisionalTargetLayers.temporal) + { + continue; + } + + requiredBitrate = producerRtpStream->GetLayerBitrate(nowMs, 0, temporalLayer); + + // This is simulcast so we must substract the bitrate of the current + // temporal spatial layer if this is the temporal layer 0 of a higher + // spatial layer. + if ( + requiredBitrate && temporalLayer == 0 && this->provisionalTargetLayers.spatial > -1 && + spatialLayer > this->provisionalTargetLayers.spatial) + { + // TODO + MS_DUMP( + "---- 3 this->provisionalTargetLayers.spatial:%" PRIi16 ", producerRtpStreams.size():%zu", + this->provisionalTargetLayers.spatial, + this->producerRtpStreams.size()); + + auto* provisionalProducerRtpStream = + this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); + auto provisionalRequiredBitrate = provisionalProducerRtpStream->GetBitrate( + nowMs, 0, this->provisionalTargetLayers.temporal); + + if (requiredBitrate > provisionalRequiredBitrate) + { + requiredBitrate -= provisionalRequiredBitrate; + } + else + { + requiredBitrate = 1u; // Don't set 0 since it would be ignored. + } + } + + MS_DEBUG_DEV( + "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 + ", required bitrate:%" PRIu32 "]", + spatialLayer, + temporalLayer, + virtualBitrate, + requiredBitrate); + + // If active layer, end iterations here. Otherwise move to next spatial + // layer. + if (requiredBitrate) + { + goto done; + } + else + { + break; + } + } + + // If this is the preferred spatial layer or higher, take it and exit. + if (spatialLayer >= this->preferredLayers.spatial) + { + break; + } + } + + done: + + // No higher active layers found. + if (!requiredBitrate) + { + return 0u; + } + + // No luck. + if (requiredBitrate > virtualBitrate) + { + return 0u; + } + + // Set provisional layers. + this->provisionalTargetLayers.spatial = spatialLayer; + this->provisionalTargetLayers.temporal = temporalLayer; + + MS_DEBUG_DEV( + "setting provisional layers to %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 + ", required bitrate:%" PRIu32 "]", + this->provisionalTargetLayers.spatial, + this->provisionalTargetLayers.temporal, + virtualBitrate, + requiredBitrate); + + if (requiredBitrate <= bitrate) + { + return requiredBitrate; + } + else if (requiredBitrate <= virtualBitrate) + { + return bitrate; + } + else + { + return requiredBitrate; // NOTE: This cannot happen. + } + } + + void MultiStreamConsumer::ApplyLayers() + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + MS_ASSERT(this->kind == RTC::Media::Kind::VIDEO, "should be video"); + MS_ASSERT(IsActive(), "should be active"); + + auto provisionalTargetLayers = this->provisionalTargetLayers; + + // Reset provisional target layers. + this->provisionalTargetLayers.Reset(); + + if (provisionalTargetLayers != this->targetLayers) + { + UpdateTargetLayers(provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); + + // If this looks like a spatial layer downgrade due to BWE limitations, set + // member. + if ( + this->rtpStream->GetActiveMs() > BweDowngradeMinActiveMs && + this->targetLayers.spatial < this->currentSpatialLayer && + this->currentSpatialLayer <= this->preferredLayers.spatial) + { + MS_DEBUG_DEV( + "possible target spatial layer downgrade (from %" PRIi16 " to %" PRIi16 + ") due to BWE limitation", + this->currentSpatialLayer, + this->targetLayers.spatial); + + this->lastBweDowngradeAtMs = DepLibUV::GetTimeMs(); + } + } + } + + uint32_t MultiStreamConsumer::GetDesiredBitrate() const + { + MS_TRACE(); + + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + + // Audio SimpleConsumer does not play the BWE game. + if (this->kind != RTC::Media::Kind::VIDEO) + { + return 0u; + } + + if (!IsActive()) + { + return 0u; + } + + auto nowMs = DepLibUV::GetTimeMs(); + uint32_t desiredBitrate{ 0u }; + + // Let's iterate all streams of the Producer (from highest to lowest) and + // obtain their bitrate. Choose the highest one. + // NOTE: When the Producer enables a higher stream, initially the bitrate of + // it could be less than the bitrate of a lower stream. That's why we + // iterate all streams here anyway. + for (auto sIdx{ static_cast(this->producerRtpStreams.size() - 1) }; sIdx >= 0; --sIdx) + { + // TODO + MS_DUMP( + "---- 4 sIdx:%" PRIi16 ", producerRtpStreams.size():%zu", + sIdx, + this->producerRtpStreams.size()); + + auto* producerRtpStream = this->producerRtpStreams.at(sIdx); + + if (!producerRtpStream) + { + continue; + } + + auto streamBitrate = producerRtpStream->GetBitrate(nowMs); + + desiredBitrate = std::max(streamBitrate, desiredBitrate); + } + + // If consumer.rtpParameters.encodings[0].maxBitrate was given and it's + // greater than computed one, then use it. + auto maxBitrate = this->rtpParameters.encodings[0].maxBitrate; + + desiredBitrate = std::max(maxBitrate, desiredBitrate); + + return desiredBitrate; + } + + // NOLINTNEXTLINE(misc-no-recursion) + void MultiStreamConsumer::SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) + { + MS_TRACE(); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.consumerId = this->id; +#endif + + // TODO + MS_DUMP( + "---- 5 packet->GetSsrc():%" PRIu32 ", mapMappedSsrcSpatialLayer.size():%zu", + packet->GetSsrc(), + this->mapMappedSsrcSpatialLayer.size()); + + // TODO + MS_DUMP("------ 5.1 mapMappedSsrcSpatialLayer>"); + for (const auto& kv : this->mapMappedSsrcSpatialLayer) + { + MS_DUMP("------------ mappeSsrc:%" PRIu32 ", spatialLayer:%" PRIi16, kv.first, kv.second); + } + MS_DUMP("------ 5.1 mapMappedSsrcSpatialLayer>"); + + auto spatialLayer = this->mapMappedSsrcSpatialLayer.at(packet->GetSsrc()); + + // TODO + MS_DUMP("----- 5.2 OK!!!"); + + if (!IsActive()) + { + // Only drop the packet in the RTP sequence manager if it belongs to the + // current spatial layer. + if (spatialLayer == this->currentSpatialLayer) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::CONSUMER_INACTIVE); +#endif + + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); + } + + return; + } + + if (this->targetLayers.temporal == -1) + { + // Only drop the packet in the RTP sequence manager if it belongs to the + // current spatial layer. + if (spatialLayer == this->currentSpatialLayer) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::INVALID_TARGET_LAYER); +#endif + + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); + } + + return; + } + + auto payloadType = packet->GetPayloadType(); + + // NOTE: This may happen if this Consumer supports just some codecs of those + // in the corresponding Producer. + if (!this->supportedCodecPayloadTypes[payloadType]) + { + // Only drop the packet in the RTP sequence manager if it belongs to the + // current spatial layer. + if (spatialLayer == this->currentSpatialLayer) + { + MS_WARN_DEV("payload type not supported [payloadType:%" PRIu8 "]", payloadType); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::UNSUPPORTED_PAYLOAD_TYPE); +#endif + + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); + } + + return; + } + + bool shouldSwitchCurrentSpatialLayer{ false }; + + // Check whether this is the packet we are waiting for in order to update + // the current spatial layer. + if ( + this->currentSpatialLayer != this->targetLayers.spatial && + spatialLayer == this->targetLayers.spatial) + { + // Ignore if not a key frame. + if (this->keyFrameSupported && !packet->IsKeyFrame()) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); +#endif + + // NOTE: Don't drop the packet in the RTP sequence manager since this + // packet doesn't belong to the current spatial layer. + + // Store the packet for the scenario in which this packet is part of the + // key frame and it arrived before the first packet of the key frame. + StorePacketInTargetLayerRetransmissionBuffer(packet, sharedPacket); + + return; + } + + shouldSwitchCurrentSpatialLayer = true; + + // Need to resync the stream. + this->syncRequired = true; + this->spatialLayerToSync = spatialLayer; + } + // If the packet belongs to different spatial layer than the one being sent, + // drop it. + else if (spatialLayer != this->currentSpatialLayer) + { + // NOTE: Don't drop the packet in the RTP sequence manager since this + // packet doesn't belong to the current spatial layer. + + return; + } + + // If we need to sync and this is not a key frame, ignore the packet. + // NOTE: syncRequired is true if packet is a key frame of the target spatial + // layer or if transport just connected or consumer resumed. + if (this->syncRequired && this->keyFrameSupported && !packet->IsKeyFrame()) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); +#endif + + // NOTE: No need to drop the packet in the RTP sequence manager since here + // we are blocking all packets but the key frame that would trigger sync + // below. + + // Store the packet for the scenario in which this packet is part of the + // key frame and it arrived before the first packet of the key frame. + StorePacketInTargetLayerRetransmissionBuffer(packet, sharedPacket); + + return; + } + + // Packets with only padding are not forwarded. + if (packet->GetPayloadLength() == 0) + { + // Only drop the packet in the RTP sequence manager if it belongs to the + // current spatial layer. + if (spatialLayer == this->currentSpatialLayer) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); +#endif + + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); + } + + return; + } + + // Whether this is the first packet after re-sync. + const bool isSyncPacket = this->syncRequired; + + // Whether packets stored in the target layer retransmission buffer must be + // sent once this packet is sent. + bool sendPacketsInTargetLayerRetransmissionBuffer{ false }; + + // Sync sequence number and timestamp if required. + if (isSyncPacket && (this->spatialLayerToSync == -1 || spatialLayer == this->spatialLayerToSync)) + { + if (packet->IsKeyFrame()) + { + MS_DEBUG_TAG( + rtp, + "sync key frame received [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + sendPacketsInTargetLayerRetransmissionBuffer = true; + } + + uint32_t tsOffset{ 0u }; + + // Sync our RTP stream's RTP timestamp. + if (spatialLayer == this->tsReferenceSpatialLayer) + { + tsOffset = 0u; + } + // If this is not the RTP stream we use as TS reference, do NTP based RTP + // TS synchronization. + else + { + auto* producerTsReferenceRtpStream = GetProducerTsReferenceRtpStream(); + auto* producerTargetRtpStream = GetProducerTargetRtpStream(); + + // NOTE: If we are here is because we have Sender Reports for both the + // TS reference stream and the target one. + MS_ASSERT( + producerTsReferenceRtpStream->GetSenderReportNtpMs(), + "no Sender Report for TS reference RTP stream"); + MS_ASSERT( + producerTargetRtpStream->GetSenderReportNtpMs(), "no Sender Report for current RTP stream"); + + // Calculate NTP and TS stuff. + auto ntpMs1 = producerTsReferenceRtpStream->GetSenderReportNtpMs(); + auto ts1 = producerTsReferenceRtpStream->GetSenderReportTs(); + auto ntpMs2 = producerTargetRtpStream->GetSenderReportNtpMs(); + auto ts2 = producerTargetRtpStream->GetSenderReportTs(); + int64_t diffMs; + + if (ntpMs2 >= ntpMs1) + { + diffMs = ntpMs2 - ntpMs1; + } + else + { + diffMs = -1 * (ntpMs1 - ntpMs2); + } + + const int64_t diffTs = diffMs * this->rtpStream->GetClockRate() / 1000; + const uint32_t newTs2 = ts2 - diffTs; + + // Apply offset. This is the difference that later must be removed from the + // sending RTP packet. + tsOffset = newTs2 - ts1; + } + + // When switching to a new stream it may happen that the timestamp of this + // key frame is lower than the highest timestamp sent to the remote endpoint. + // If so, apply an extra offset to "fix" it for the whole live of this + // selected Producer stream. + if (shouldSwitchCurrentSpatialLayer && (packet->GetTimestamp() - tsOffset <= this->rtpStream->GetMaxPacketTs())) + { + // Max delay in ms we allow for the stream when switching. + // https://en.wikipedia.org/wiki/Audio-to-video_synchronization#Recommendations + static const uint32_t MaxExtraOffsetMs{ 75u }; + + // Outgoing packet matches the highest timestamp seen in the previous + // stream. Apply an expected offset for a new frame in a 30fps stream. + static const uint8_t MsOffset{ 33u }; // (1 / 30 * 1000). + + const int64_t maxTsExtraOffset = MaxExtraOffsetMs * this->rtpStream->GetClockRate() / 1000; + uint32_t tsExtraOffset = this->rtpStream->GetMaxPacketTs() - packet->GetTimestamp() + + tsOffset + (MsOffset * this->rtpStream->GetClockRate() / 1000); + + // NOTE: Don't ask for a key frame if already done. + if (this->keyFrameForTsOffsetRequested) + { + // Give up and use the theoretical offset. + if (tsExtraOffset > maxTsExtraOffset) + { + MS_WARN_TAG( + simulcast, + "giving up on proper stream switching after got a requested keyframe for which still too high RTP timestamp extra offset is needed (%" PRIu32 + ")", + tsExtraOffset); + + tsExtraOffset = 1u; + } + } + else if (tsExtraOffset > maxTsExtraOffset) + { + MS_WARN_TAG( + simulcast, + "cannot switch stream due to too high RTP timestamp extra offset needed (%" PRIu32 + "), requesting keyframe", + tsExtraOffset); + + RequestKeyFrameForTargetSpatialLayer(); + + this->keyFrameForTsOffsetRequested = true; + + // Reset flags since we are discarding this key frame. + this->syncRequired = false; + this->spatialLayerToSync = -1; + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded( + RTC::RtcLogger::RtpPacket::DiscardReason::TOO_HIGH_TIMESTAMP_EXTRA_NEEDED); +#endif + + // NOTE: Don't drop the packet in the RTP sequence manager since this + // packet doesn't belong to the current spatial layer. + + return; + } + + if (tsExtraOffset > 0u) + { + MS_DEBUG_TAG( + simulcast, + "RTP timestamp extra offset generated for stream switching: %" PRIu32, + tsExtraOffset); + + // Increase the timestamp offset for the whole life of this Producer stream + // (until switched to a different one). + tsOffset -= tsExtraOffset; + } + } + + this->tsOffset = tsOffset; + + // Sync our RTP stream's sequence number. + // If previous frame has not been sent completely when we switch layer, + // we can tell libwebrtc that previous frame is incomplete by skipping + // one RTP sequence number. + // 'packet->GetSequenceNumber() -2' may increase SeqManager::base and + // increase the output sequence number. + // https://github.com/versatica/mediasoup/issues/408 + this->rtpSeqManager.Sync(packet->GetSequenceNumber() - (this->lastSentPacketHasMarker ? 1 : 2)); + + if (this->encodingContext) + { + this->encodingContext->SyncRequired(); + } + + this->syncRequired = false; + this->spatialLayerToSync = -1; + this->keyFrameForTsOffsetRequested = false; + } + + if (!shouldSwitchCurrentSpatialLayer && this->checkingForOldPacketsInSpatialLayer) + { + // If this is a packet previous to the spatial layer switch, ignore the + // packet. + // NOTE: We drop it in RTP sequence manager because this packet belongs + // to current spatial layer. + if (SeqManager::IsSeqLowerThan( + packet->GetSequenceNumber(), this->snReferenceSpatialLayer)) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded( + RTC::RtcLogger::RtpPacket::DiscardReason::PACKET_PREVIOUS_TO_SPATIAL_LAYER_SWITCH); +#endif + + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); + + return; + } + else if (SeqManager::IsSeqHigherThan( + packet->GetSequenceNumber(), this->snReferenceSpatialLayer + MaxSequenceNumberGap)) + { + this->checkingForOldPacketsInSpatialLayer = false; + } + } + + bool marker{ false }; + + if (shouldSwitchCurrentSpatialLayer) + { + // Update current spatial layer. + this->currentSpatialLayer = this->targetLayers.spatial; + + this->snReferenceSpatialLayer = packet->GetSequenceNumber(); + this->checkingForOldPacketsInSpatialLayer = true; + + // Update target and current temporal layer. + if (this->encodingContext) + { + this->encodingContext->SetTargetTemporalLayer(this->targetLayers.temporal); + this->encodingContext->SetCurrentTemporalLayer(packet->GetTemporalLayer()); + } + + // Reset the score of our RtpStream to 10. + this->rtpStream->ResetScore(10u, /*notify*/ false); + + // Emit the layersChange event. + EmitLayersChange(); + + // Emit the score event. + EmitScore(); + + // Rewrite payload if needed. + if (this->encodingContext) + { + packet->ProcessPayload(this->encodingContext.get(), marker); + } + } + else if (this->encodingContext) + { + auto previousTemporalLayer = this->encodingContext->GetCurrentTemporalLayer(); + + // Rewrite payload if needed. Drop packet if necessary. + // NOTE: We drop it in RTP sequence manager because this packet belongs + // to current spatial layer. + if (!packet->ProcessPayload(this->encodingContext.get(), marker)) + { +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::DROPPED_BY_CODEC); +#endif + + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); + + return; + } + + if (previousTemporalLayer != this->encodingContext->GetCurrentTemporalLayer()) + { + EmitLayersChange(); + } + } + + // Update RTP seq number and timestamp based on NTP offset. + uint16_t seq; + const uint32_t timestamp = packet->GetTimestamp() - this->tsOffset; + + this->rtpSeqManager.Input(packet->GetSequenceNumber(), seq); + + // Save original packet fields. + auto origSsrc = packet->GetSsrc(); + auto origSeq = packet->GetSequenceNumber(); + auto origTimestamp = packet->GetTimestamp(); + + // Rewrite packet. + packet->SetSsrc(this->rtpParameters.encodings[0].ssrc); + packet->SetSequenceNumber(seq); + packet->SetTimestamp(timestamp); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.sendRtpTimestamp = timestamp; + packet->logger.sendSeqNumber = seq; +#endif + + if (isSyncPacket) + { + MS_DEBUG_TAG( + rtp, + "sending sync packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 + "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp(), + origSsrc, + origSeq, + origTimestamp); + } + + const RTC::RTP::RtpStreamSend::ReceivePacketResult result = + this->rtpStream->ReceivePacket(packet, sharedPacket); + + if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) + { + if (this->rtpSeqManager.GetMaxOutput() == packet->GetSequenceNumber()) + { + this->lastSentPacketHasMarker = packet->HasMarker(); + } + + // Send the packet. + this->listener->OnConsumerSendRtpPacket(this, packet); + + // May emit 'trace' event. + EmitTraceEventRtpAndKeyFrameTypes(packet); + } + else + { + MS_WARN_TAG( + rtp, + "failed to send packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 + "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp(), + origSsrc, + origSeq, + origTimestamp); + +#ifdef MS_RTC_LOGGER_RTP + packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::SEND_RTP_STREAM_DISCARDED); +#endif + } + + // Restore packet fields. + packet->SetSsrc(origSsrc); + packet->SetSequenceNumber(origSeq); + packet->SetTimestamp(origTimestamp); + + // Restore the original payload if needed. + packet->RestorePayload(); + + // If sharedPacket doesn't have a packet inside and it has been stored we + // need to clone the packet into it. + if (!sharedPacket.HasPacket() && result == RTC::RTP::RtpStreamSend::ReceivePacketResult::ACCEPTED_AND_STORED) + { + sharedPacket.Assign(packet); + } + + // If sent packet was the first packet of a key frame, let's send buffered + // packets belonging to the same key frame that arrived earlier due to + // packet misorder. + if (sendPacketsInTargetLayerRetransmissionBuffer) + { + // NOTE: Only send buffered packets if the first packet containing the key + // frame was sent. + if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) + { + for (auto& kv : this->targetLayerRetransmissionBuffer) + { + auto& bufferedSharedPacket = kv.second; + auto* bufferedPacket = bufferedSharedPacket.GetPacket(); + + if (bufferedPacket->GetSequenceNumber() > origSeq) + { + MS_DEBUG_DEV( + "sending packet buffered in the target layer retransmission buffer [ssrc:%" PRIu32 + ", seq:%" PRIu16 ", ts:%" PRIu32 + "] after sending first packet of the key frame [ssrc:%" PRIu32 ", seq:%" PRIu16 + ", ts:%" PRIu32 "]", + bufferedPacket->GetSsrc(), + bufferedPacket->GetSequenceNumber(), + bufferedPacket->GetTimestamp(), + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + SendRtpPacket(bufferedPacket, bufferedSharedPacket); + + // Be sure that the target layer retransmission buffer has not been + // emptied as a result of sending this packet. If so, exit the loop. + if (this->targetLayerRetransmissionBuffer.empty()) + { + MS_DEBUG_DEV( + "target layer retransmission buffer emptied while iterating it, exiting the loop"); + + break; + } + } + } + } + + this->targetLayerRetransmissionBuffer.clear(); + } + } + + bool MultiStreamConsumer::GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) + { + MS_TRACE(); + + if (static_cast((nowMs - this->lastRtcpSentTime) * 1.15) < this->maxRtcpInterval) + { + return true; + } + + auto* senderReport = this->rtpStream->GetRtcpSenderReport(nowMs); + + if (!senderReport) + { + return true; + } + + // Build SDES chunk for this sender. + auto* sdesChunk = this->rtpStream->GetRtcpSdesChunk(); + + auto* delaySinceLastRrSsrcInfo = this->rtpStream->GetRtcpXrDelaySinceLastRrSsrcInfo(nowMs); + + // RTCP Compound packet buffer cannot hold the data. + if (!packet->Add(senderReport, sdesChunk, delaySinceLastRrSsrcInfo)) + { + return false; + } + + this->lastRtcpSentTime = nowMs; + + return true; + } + + void MultiStreamConsumer::NeedWorstRemoteFractionLost( + uint32_t /*mappedSsrc*/, uint8_t& worstRemoteFractionLost) + { + MS_TRACE(); + + if (!IsActive()) + { + return; + } + + auto fractionLost = this->rtpStream->GetFractionLost(); + + // If our fraction lost is worse than the given one, update it. + worstRemoteFractionLost = std::max(fractionLost, worstRemoteFractionLost); + } + + void MultiStreamConsumer::ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) + { + MS_TRACE(); + + if (!IsActive()) + { + return; + } + + // May emit 'trace' event. + EmitTraceEventNackType(); + + this->rtpStream->ReceiveNack(nackPacket); + } + + void MultiStreamConsumer::ReceiveKeyFrameRequest( + RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) + { + MS_TRACE(); + + switch (messageType) + { + case RTC::RTCP::FeedbackPs::MessageType::PLI: + { + EmitTraceEventPliType(ssrc); + + break; + } + + case RTC::RTCP::FeedbackPs::MessageType::FIR: + { + EmitTraceEventFirType(ssrc); + + break; + } + + default:; + } + + this->rtpStream->ReceiveKeyFrameRequest(messageType); + + if (IsActive()) + { + RequestKeyFrameForCurrentSpatialLayer(); + } + } + + void MultiStreamConsumer::ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) + { + MS_TRACE(); + + this->rtpStream->ReceiveRtcpReceiverReport(report); + } + + void MultiStreamConsumer::ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) + { + MS_TRACE(); + + this->rtpStream->ReceiveRtcpXrReceiverReferenceTime(report); + } + + uint32_t MultiStreamConsumer::GetTransmissionRate(uint64_t nowMs) + { + MS_TRACE(); + + if (!IsActive()) + { + return 0u; + } + + return this->rtpStream->GetBitrate(nowMs); + } + + float MultiStreamConsumer::GetRtt() const + { + MS_TRACE(); + + return this->rtpStream->GetRtt(); + } + + void MultiStreamConsumer::UserOnTransportConnected() + { + MS_TRACE(); + + this->syncRequired = true; + this->spatialLayerToSync = -1; + this->keyFrameForTsOffsetRequested = false; + + if (IsActive()) + { + MayChangeLayers(); + } + } + + void MultiStreamConsumer::UserOnTransportDisconnected() + { + MS_TRACE(); + + this->lastBweDowngradeAtMs = 0u; + + this->rtpStream->Pause(); + this->targetLayerRetransmissionBuffer.clear(); + + UpdateTargetLayers(-1, -1); + } + + void MultiStreamConsumer::UserOnPaused() + { + MS_TRACE(); + + this->lastBweDowngradeAtMs = 0u; + + this->rtpStream->Pause(); + this->targetLayerRetransmissionBuffer.clear(); + + UpdateTargetLayers(-1, -1); + + if (this->externallyManagedBitrate) + { + this->listener->OnConsumerNeedZeroBitrate(this); + } + } + + void MultiStreamConsumer::UserOnResumed() + { + MS_TRACE(); + + this->syncRequired = true; + this->spatialLayerToSync = -1; + this->keyFrameForTsOffsetRequested = false; + this->checkingForOldPacketsInSpatialLayer = false; + + if (IsActive()) + { + MayChangeLayers(); + } + } + + void MultiStreamConsumer::CreateRtpStream() + { + MS_TRACE(); + + auto& encoding = this->rtpParameters.encodings[0]; + const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); + + MS_DEBUG_TAG( + rtp, "[ssrc:%" PRIu32 ", payloadType:%" PRIu8 "]", encoding.ssrc, mediaCodec->payloadType); + + // Set stream params. + RTC::RTP::RtpStream::Params params; + + params.ssrc = encoding.ssrc; + params.payloadType = mediaCodec->payloadType; + params.mimeType = mediaCodec->mimeType; + params.clockRate = mediaCodec->clockRate; + params.cname = this->rtpParameters.rtcp.cname; + params.spatialLayers = encoding.spatialLayers; + params.temporalLayers = encoding.temporalLayers; + + // Check in band FEC in codec parameters. + if (mediaCodec->parameters.HasInteger("useinbandfec") && mediaCodec->parameters.GetInteger("useinbandfec") == 1) + { + MS_DEBUG_TAG(rtp, "in band FEC enabled"); + + params.useInBandFec = true; + } + + // Check DTX in codec parameters. + if (mediaCodec->parameters.HasInteger("usedtx") && mediaCodec->parameters.GetInteger("usedtx") == 1) + { + MS_DEBUG_TAG(rtp, "DTX enabled"); + + params.useDtx = true; + } + + // Check DTX in the encoding. + if (encoding.dtx) + { + MS_DEBUG_TAG(rtp, "DTX enabled"); + + params.useDtx = true; + } + + for (const auto& fb : mediaCodec->rtcpFeedback) + { + if (!params.useNack && fb.type == "nack" && fb.parameter.empty()) + { + MS_DEBUG_2TAGS(rtp, rtcp, "NACK supported"); + + params.useNack = true; + } + else if (!params.usePli && fb.type == "nack" && fb.parameter == "pli") + { + MS_DEBUG_2TAGS(rtp, rtcp, "PLI supported"); + + params.usePli = true; + } + else if (!params.useFir && fb.type == "ccm" && fb.parameter == "fir") + { + MS_DEBUG_2TAGS(rtp, rtcp, "FIR supported"); + + params.useFir = true; + } + } + + this->rtpStream = new RTC::RTP::RtpStreamSend(this, params, this->rtpParameters.mid); + this->rtpStreams.push_back(this->rtpStream); + + // If the Consumer is paused, tell the RtpStreamSend. + if (IsPaused() || IsProducerPaused()) + { + this->rtpStream->Pause(); + } + + const auto* rtxCodec = this->rtpParameters.GetRtxCodecForEncoding(encoding); + + if (rtxCodec && encoding.hasRtx) + { + this->rtpStream->SetRtx(rtxCodec->payloadType, encoding.rtx.ssrc); + } + } + + void MultiStreamConsumer::RequestKeyFrames() + { + MS_TRACE(); + + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + auto* producerTargetRtpStream = GetProducerTargetRtpStream(); + auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); + + if (producerTargetRtpStream) + { + auto mappedSsrc = this->consumableRtpEncodings[this->targetLayers.spatial].ssrc; + + this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); + } + + if (producerCurrentRtpStream && producerCurrentRtpStream != producerTargetRtpStream) + { + auto mappedSsrc = this->consumableRtpEncodings[this->currentSpatialLayer].ssrc; + + this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); + } + } + + void MultiStreamConsumer::RequestKeyFrameForTargetSpatialLayer() + { + MS_TRACE(); + + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + auto* producerTargetRtpStream = GetProducerTargetRtpStream(); + + if (!producerTargetRtpStream) + { + return; + } + + auto mappedSsrc = this->consumableRtpEncodings[this->targetLayers.spatial].ssrc; + + this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); + } + + void MultiStreamConsumer::RequestKeyFrameForCurrentSpatialLayer() + { + MS_TRACE(); + + if (this->kind != RTC::Media::Kind::VIDEO) + { + return; + } + + auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); + + if (!producerCurrentRtpStream) + { + return; + } + + auto mappedSsrc = this->consumableRtpEncodings[this->currentSpatialLayer].ssrc; + + this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); + } + + void MultiStreamConsumer::MayChangeLayers(bool force) + { + MS_TRACE(); + + RTC::ConsumerTypes::VideoLayers newTargetLayers; + + if (RecalculateTargetLayers(newTargetLayers)) + { + // If bitrate externally managed, don't bother the transport unless + // the newTargetSpatialLayer has changed (or force is true). + // This is because, if bitrate is externally managed, the target temporal + // layer is managed by the available given bitrate so the transport + // will let us change it when it considers. + if (this->externallyManagedBitrate) + { + if (newTargetLayers.spatial != this->targetLayers.spatial || force) + { + this->listener->OnConsumerNeedBitrateChange(this); + } + } + else + { + UpdateTargetLayers(newTargetLayers.spatial, newTargetLayers.temporal); + } + } + } + + bool MultiStreamConsumer::RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const + { + MS_TRACE(); + + // Start with no layers. + newTargetLayers.Reset(); + + auto nowMs = DepLibUV::GetTimeMs(); + + for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx) + { + auto spatialLayer = static_cast(sIdx); + + // TODO + MS_DUMP("---- 6 sIdx:%zu, producerRtpStreams.size():%zu", sIdx, this->producerRtpStreams.size()); + + auto* producerRtpStream = this->producerRtpStreams.at(sIdx); + auto producerScore = producerRtpStream ? producerRtpStream->GetScore() : 0u; + + // If this is higher than current spatial layer and we moved to to current spatial + // layer due to BWE limitations, check how much it has elapsed since then. + if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) + { + if (newTargetLayers.spatial > -1 && spatialLayer > this->currentSpatialLayer) + { + continue; + } + } + + // Ignore spatial layers for non existing Producer streams or for those + // with score 0. + if (producerScore == 0u) + { + continue; + } + + // If the stream has not been active time enough and we have an active one + // already, move to the next spatial layer. + // NOTE: Require bitrate externally managed for this. + if (this->externallyManagedBitrate && newTargetLayers.spatial != -1 && producerRtpStream->GetActiveMs() < StreamMinActiveMs) + { + continue; + } + + // We may not yet switch to this spatial layer. + if (!CanSwitchToSpatialLayer(spatialLayer)) + { + continue; + } + + newTargetLayers.spatial = spatialLayer; + + // If this is the preferred or higher spatial layer take it and exit. + if (spatialLayer >= this->preferredLayers.spatial) + { + break; + } + } + + if (newTargetLayers.spatial != -1) + { + if (newTargetLayers.spatial == this->preferredLayers.spatial) + { + newTargetLayers.temporal = this->preferredLayers.temporal; + } + else if (newTargetLayers.spatial < this->preferredLayers.spatial) + { + newTargetLayers.temporal = static_cast(this->rtpStream->GetTemporalLayers() - 1); + } + else + { + newTargetLayers.temporal = 0; + } + } + + // Return true if any target layer changed. + return (newTargetLayers != this->targetLayers); + } + + void MultiStreamConsumer::UpdateTargetLayers( + int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer) + { + MS_TRACE(); + + // If we don't have yet a RTP timestamp reference, set it now. + if ( + newTargetSpatialLayer != -1 && (this->tsReferenceSpatialLayer == -1 || + !GetProducerTsReferenceRtpStream()->GetSenderReportNtpMs())) + { + MS_DEBUG_TAG( + simulcast, "using spatial layer %" PRIi16 " as RTP timestamp reference", newTargetSpatialLayer); + + this->tsReferenceSpatialLayer = newTargetSpatialLayer; + } + + // If the new target spatial layer doesn't match the current one, clear the + // target layer retransmission buffer. + if (newTargetSpatialLayer != this->targetLayers.spatial) + { + this->targetLayerRetransmissionBuffer.clear(); + } + + if (newTargetSpatialLayer == -1) + { + // Unset current and target layers. + this->targetLayers.spatial = -1; + this->targetLayers.temporal = -1; + this->currentSpatialLayer = -1; + + if (this->encodingContext) + { + this->encodingContext->SetTargetTemporalLayer(-1); + this->encodingContext->SetCurrentTemporalLayer(-1); + } + + MS_DEBUG_TAG( + simulcast, "target layers changed [spatial:-1, temporal:-1, consumerId:%s]", this->id.c_str()); + + EmitLayersChange(); + + return; + } + + this->targetLayers.spatial = newTargetSpatialLayer; + this->targetLayers.temporal = newTargetTemporalLayer; + + // If the new target spatial layer matches the current one, apply the new + // target temporal layer now. + if (this->encodingContext && this->targetLayers.spatial == this->currentSpatialLayer) + { + this->encodingContext->SetTargetTemporalLayer(this->targetLayers.temporal); + } + + MS_DEBUG_TAG( + simulcast, + "target layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", + this->targetLayers.spatial, + this->targetLayers.temporal, + this->id.c_str()); + + // If the target spatial layer is different than the current one, request + // a key frame. + if (this->targetLayers.spatial != this->currentSpatialLayer) + { + RequestKeyFrameForTargetSpatialLayer(); + } + } + + bool MultiStreamConsumer::CanSwitchToSpatialLayer(int16_t spatialLayer) const + { + MS_TRACE(); + + // TODO + MS_DUMP( + "---- 7 spatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", + spatialLayer, + this->producerRtpStreams.size()); + + // This method assumes that the caller has verified that there is a valid + // Producer RtpStream for the given spatial layer. + MS_ASSERT( + this->producerRtpStreams.size() > spatialLayer, + "no Producer RtpStream for the given spatialLayer:%" PRIi16, + spatialLayer); + + // We can switch to the given spatial layer if: + // - we don't have any TS reference spatial layer yet, or + // - the given spatial layer matches the TS reference spatial layer, or + // - both , the RTP streams of our TS reference spatial layer and the given + // spatial layer, have Sender Report. + return ( + this->tsReferenceSpatialLayer == -1 || spatialLayer == this->tsReferenceSpatialLayer || + this->producerRtpStreams.at(spatialLayer)->GetSenderReportNtpMs()); + } + + void MultiStreamConsumer::StorePacketInTargetLayerRetransmissionBuffer( + RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) + { + MS_TRACE(); + + MS_DEBUG_DEV( + "storing packet in target layer retransmission buffer [ssrc:%" PRIu32 ", seq:%" PRIu16 + ", ts:%" PRIu32 "]", + packet->GetSsrc(), + packet->GetSequenceNumber(), + packet->GetTimestamp()); + + // Store original packet into the buffer. Only clone once and only if + // necessary. + if (!sharedPacket.HasPacket()) + { + sharedPacket.Assign(packet); + } + // Assert that, if sharedPacket was already filled, both packet and + // sharedPacket are the very same RTP packet. + else + { + sharedPacket.AssertSamePacket(packet); + } + + this->targetLayerRetransmissionBuffer[packet->GetSequenceNumber()] = sharedPacket; + + if (this->targetLayerRetransmissionBuffer.size() > TargetLayerRetransmissionBufferSize) + { + this->targetLayerRetransmissionBuffer.erase(this->targetLayerRetransmissionBuffer.begin()); + } + } + + void MultiStreamConsumer::EmitScore() const + { + MS_TRACE(); + + auto scoreOffset = FillBufferScore(this->shared->channelNotifier->GetBufferBuilder()); + + auto notificationOffset = FBS::Consumer::CreateScoreNotification( + this->shared->channelNotifier->GetBufferBuilder(), scoreOffset); + + this->shared->channelNotifier->Emit( + this->id, + FBS::Notification::Event::CONSUMER_SCORE, + FBS::Notification::Body::Consumer_ScoreNotification, + notificationOffset); + } + + void MultiStreamConsumer::EmitLayersChange() const + { + MS_TRACE(); + + MS_DEBUG_DEV( + "current layers changed to [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", + this->currentSpatialLayer, + this->encodingContext ? this->encodingContext->GetCurrentTemporalLayer() : 1, + this->id.c_str()); + + flatbuffers::Offset layersOffset; + + if (this->currentSpatialLayer >= 0) + { + layersOffset = FBS::Consumer::CreateConsumerLayers( + this->shared->channelNotifier->GetBufferBuilder(), + this->currentSpatialLayer, + this->encodingContext ? this->encodingContext->GetCurrentTemporalLayer() : 1); + } + + auto notificationOffset = FBS::Consumer::CreateLayersChangeNotification( + this->shared->channelNotifier->GetBufferBuilder(), layersOffset); + + this->shared->channelNotifier->Emit( + this->id, + FBS::Notification::Event::CONSUMER_LAYERS_CHANGE, + FBS::Notification::Body::Consumer_LayersChangeNotification, + notificationOffset); + } + + RTC::RTP::RtpStreamRecv* MultiStreamConsumer::GetProducerCurrentRtpStream() const + { + MS_TRACE(); + + if (this->currentSpatialLayer == -1) + { + return nullptr; + } + + // TODO + MS_DUMP( + "---- 8 this->currentSpatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", + this->currentSpatialLayer, + this->producerRtpStreams.size()); + + // This may return nullptr. + return this->producerRtpStreams.at(this->currentSpatialLayer); + } + + RTC::RTP::RtpStreamRecv* MultiStreamConsumer::GetProducerTargetRtpStream() const + { + MS_TRACE(); + + if (this->targetLayers.spatial == -1) + { + return nullptr; + } + + // TODO + MS_DUMP( + "---- 9 this->targetLayers.spatial:%" PRIi16 ", producerRtpStreams.size():%zu", + this->targetLayers.spatial, + this->producerRtpStreams.size()); + + // This may return nullptr. + return this->producerRtpStreams.at(this->targetLayers.spatial); + } + + RTC::RTP::RtpStreamRecv* MultiStreamConsumer::GetProducerTsReferenceRtpStream() const + { + MS_TRACE(); + + if (this->tsReferenceSpatialLayer == -1) + { + return nullptr; + } + + // TODO + MS_DUMP( + "---- 10 this->tsReferenceSpatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", + this->tsReferenceSpatialLayer, + this->producerRtpStreams.size()); + + // This may return nullptr. + return this->producerRtpStreams.at(this->tsReferenceSpatialLayer); + } + + void MultiStreamConsumer::OnRtpStreamScore( + RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) + { + MS_TRACE(); + + // Emit the score event. + EmitScore(); + + if (IsActive()) + { + // Just check target layers if our bitrate is not externally managed. + // NOTE: For now this is a bit useless since, when locally managed, we do + // not check the Consumer score at all. + if (!this->externallyManagedBitrate) + { + MayChangeLayers(); + } + } + } + + void MultiStreamConsumer::OnRtpStreamRetransmitRtpPacket( + RTC::RTP::RtpStreamSend* /*rtpStream*/, RTC::RTP::Packet* packet) + { + MS_TRACE(); + + this->listener->OnConsumerRetransmitRtpPacket(this, packet); + + // May emit 'trace' event. + EmitTraceEventRtpAndKeyFrameTypes(packet, this->rtpStream->HasRtx()); + } +} // namespace RTC diff --git a/worker/src/RTC/SimpleConsumer.cpp b/worker/src/RTC/SimpleConsumer.cpp index 3b3962e6ff..4f710136e8 100644 --- a/worker/src/RTC/SimpleConsumer.cpp +++ b/worker/src/RTC/SimpleConsumer.cpp @@ -1,13 +1,12 @@ -#include "FBS/consumer.h" #define MS_CLASS "RTC::SimpleConsumer" // #define MS_LOG_DEV_LEVEL 3 +#include "RTC/SimpleConsumer.hpp" #include "DepLibUV.hpp" #include "Logger.hpp" #include "MediaSoupErrors.hpp" #include "Utils.hpp" #include "RTC/RTP/Codecs/Tools.hpp" -#include "RTC/SimpleConsumer.hpp" #ifdef MS_RTC_LOGGER_RTP #include "RTC/RtcLogger.hpp" #endif diff --git a/worker/src/RTC/Transport.cpp b/worker/src/RTC/Transport.cpp index ec9dac0038..643c8efe14 100644 --- a/worker/src/RTC/Transport.cpp +++ b/worker/src/RTC/Transport.cpp @@ -12,6 +12,7 @@ #include "FBS/transport.h" #include "RTC/BweType.hpp" #include "RTC/Consts.hpp" +#include "RTC/MultiStreamConsumer.hpp" #include "RTC/PipeConsumer.hpp" #include "RTC/RTCP/FeedbackPs.hpp" #include "RTC/RTCP/FeedbackPsAfb.hpp" @@ -20,8 +21,6 @@ #include "RTC/RTCP/FeedbackRtpTransport.hpp" #include "RTC/RTCP/XrDelaySinceLastRr.hpp" #include "RTC/RtpDictionaries.hpp" -#include "RTC/SimpleConsumer.hpp" -#include "RTC/SimulcastConsumer.hpp" #include "RTC/SvcConsumer.hpp" #ifdef MS_RTC_LOGGER_RTP #include "RTC/RtcLogger.hpp" @@ -806,7 +805,8 @@ namespace RTC case RTC::RtpParameters::Type::SIMPLE: { // This may throw. - consumer = new RTC::SimpleConsumer(this->shared, consumerId, producerId, this, body); + consumer = new RTC::MultiStreamConsumer( + this->shared, consumerId, producerId, RTC::RtpParameters::Type::SIMPLE, this, body); break; } @@ -814,7 +814,8 @@ namespace RTC case RTC::RtpParameters::Type::SIMULCAST: { // This may throw. - consumer = new RTC::SimulcastConsumer(this->shared, consumerId, producerId, this, body); + consumer = new RTC::MultiStreamConsumer( + this->shared, consumerId, producerId, RTC::RtpParameters::Type::SIMULCAST, this, body); break; } diff --git a/worker/test/src/RTC/TestSimpleConsumer.cpp b/worker/test/src/RTC/TestMultiStreamConsumer.cpp similarity index 73% rename from worker/test/src/RTC/TestSimpleConsumer.cpp rename to worker/test/src/RTC/TestMultiStreamConsumer.cpp index 5a23c943d1..95fb67266f 100644 --- a/worker/test/src/RTC/TestSimpleConsumer.cpp +++ b/worker/test/src/RTC/TestMultiStreamConsumer.cpp @@ -3,13 +3,13 @@ #include "Channel/ChannelSocket.hpp" #include "FBS/rtpParameters.h" #include "FBS/transport.h" +#include "RTC/MultiStreamConsumer.hpp" #include "RTC/RTP/Packet.hpp" #include "RTC/RTP/RtpStream.hpp" #include "RTC/RTP/RtpStreamRecv.hpp" #include "RTC/RTP/SharedPacket.hpp" #include "RTC/RtpDictionaries.hpp" #include "RTC/Shared.hpp" -#include "RTC/SimpleConsumer.hpp" #include namespace @@ -113,7 +113,7 @@ namespace return rtpParameters.FillBuffer(builder); }; - std::unique_ptr createConsumer(ConsumerListener* listener) + std::unique_ptr createConsumer(ConsumerListener* listener) { flatbuffers::FlatBufferBuilder bufferBuilder; @@ -141,10 +141,11 @@ namespace const auto* consumeRequest = flatbuffers::GetRoot(buf); - return std::make_unique( + return std::make_unique( &shared, consumeRequest->consumerId()->str(), consumeRequest->producerId()->str(), + RTC::RtpParameters::Type::SIMPLE, listener, consumeRequest); } @@ -171,16 +172,15 @@ namespace const std::vector scores{ 10 }; consumer->ProducerRtpStreamScores(&scores); - consumer->ProducerNewRtpStream(rtpStream.get(), 1234); } std::unique_ptr listener; - std::unique_ptr consumer; + std::unique_ptr consumer; std::unique_ptr rtpStream; }; } // namespace -SCENARIO("SimpleConsumer", "[rtp][consumer]") +SCENARIO("MultiStreamConsumer", "[rtp][consumer]") { // clang-format off uint8_t buffer[] = @@ -244,79 +244,79 @@ SCENARIO("SimpleConsumer", "[rtp][consumer]") delete packet; } - SECTION("RTP packets are not forwarded for unsupported payload types") - { - Fixture fixture; + // SECTION("RTP packets are not forwarded for unsupported payload types") + // { + // Fixture fixture; - // Indicate that the transport is connected in order to activate the consumer. - dynamic_cast(fixture.consumer.get())->TransportConnected(); + // // Indicate that the transport is connected in order to activate the consumer. + // dynamic_cast(fixture.consumer.get())->TransportConnected(); - auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); - RTC::RTP::SharedPacket sharedPacket(packet); + // auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); + // RTC::RTP::SharedPacket sharedPacket(packet); - packet->SetPayloadType(payloadType + 1); + // packet->SetPayloadType(payloadType + 1); - fixture.consumer->SendRtpPacket(packet, sharedPacket); - fixture.listener->Verify(0); + // fixture.consumer->SendRtpPacket(packet, sharedPacket); + // fixture.listener->Verify(0); - delete packet; - } + // delete packet; + // } - SECTION("RTP packets with empty payload are not forwarded") - { - Fixture fixture; + // SECTION("RTP packets with empty payload are not forwarded") + // { + // Fixture fixture; - // Indicate that the transport is connected in order to activate the consumer. - dynamic_cast(fixture.consumer.get())->TransportConnected(); + // // Indicate that the transport is connected in order to activate the consumer. + // dynamic_cast(fixture.consumer.get())->TransportConnected(); - auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 0); - RTC::RTP::SharedPacket sharedPacket(packet); + // auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 0); + // RTC::RTP::SharedPacket sharedPacket(packet); - packet->SetPayloadType(payloadType + 1); + // packet->SetPayloadType(payloadType + 1); - fixture.consumer->SendRtpPacket(packet, sharedPacket); - fixture.listener->Verify(0); + // fixture.consumer->SendRtpPacket(packet, sharedPacket); + // fixture.listener->Verify(0); - delete packet; - } + // delete packet; + // } - SECTION("outgoing RTP packets are forwarded with increased sequence number") - { - Fixture fixture; + // SECTION("outgoing RTP packets are forwarded with increased sequence number") + // { + // Fixture fixture; - // Indicate that the transport is connected in order to activate the consumer. - dynamic_cast(fixture.consumer.get())->TransportConnected(); + // // Indicate that the transport is connected in order to activate the consumer. + // dynamic_cast(fixture.consumer.get())->TransportConnected(); - auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); - RTC::RTP::SharedPacket sharedPacket(packet); + // auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); + // RTC::RTP::SharedPacket sharedPacket(packet); - uint16_t seq{ 1 }; + // uint16_t seq{ 1 }; - packet->SetSequenceNumber(seq++); - packet->SetPayloadType(payloadType); - sharedPacket.Assign(packet); + // packet->SetSequenceNumber(seq++); + // packet->SetPayloadType(payloadType); + // sharedPacket.Assign(packet); - fixture.consumer->SendRtpPacket(packet, sharedPacket); + // fixture.consumer->SendRtpPacket(packet, sharedPacket); - packet->SetSequenceNumber(seq++); - sharedPacket.Assign(packet); + // packet->SetSequenceNumber(seq++); + // sharedPacket.Assign(packet); - fixture.consumer->SendRtpPacket(packet, sharedPacket); + // fixture.consumer->SendRtpPacket(packet, sharedPacket); - packet->SetSequenceNumber(seq++); - sharedPacket.Assign(packet); + // packet->SetSequenceNumber(seq++); + // sharedPacket.Assign(packet); - fixture.consumer->SendRtpPacket(packet, sharedPacket); + // fixture.consumer->SendRtpPacket(packet, sharedPacket); - packet->SetSequenceNumber(seq++); - // Remove the payload so it won't be sent. - packet->RemovePayload(); - sharedPacket.Assign(packet); + // packet->SetSequenceNumber(seq++); + // // Remove the payload so it won't be sent. + // packet->RemovePayload(); + // sharedPacket.Assign(packet); - fixture.consumer->SendRtpPacket(packet, sharedPacket); + // fixture.consumer->SendRtpPacket(packet, sharedPacket); - fixture.listener->Verify(3); + // fixture.listener->Verify(3); - delete packet; - } + // delete packet; + // } } From 0d2cc7d7357fc535c027a38ed18fe40a6a9a1be9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Mon, 9 Feb 2026 18:35:28 +0100 Subject: [PATCH 02/10] more --- worker/src/RTC/MultiStreamConsumer.cpp | 3 - .../test/src/RTC/TestMultiStreamConsumer.cpp | 102 +++++++++--------- 2 files changed, 51 insertions(+), 54 deletions(-) diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp index a84190457e..6b45fe7022 100644 --- a/worker/src/RTC/MultiStreamConsumer.cpp +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -789,9 +789,6 @@ namespace RTC auto spatialLayer = this->mapMappedSsrcSpatialLayer.at(packet->GetSsrc()); - // TODO - MS_DUMP("----- 5.2 OK!!!"); - if (!IsActive()) { // Only drop the packet in the RTP sequence manager if it belongs to the diff --git a/worker/test/src/RTC/TestMultiStreamConsumer.cpp b/worker/test/src/RTC/TestMultiStreamConsumer.cpp index 95fb67266f..77db0d1339 100644 --- a/worker/test/src/RTC/TestMultiStreamConsumer.cpp +++ b/worker/test/src/RTC/TestMultiStreamConsumer.cpp @@ -187,7 +187,7 @@ SCENARIO("MultiStreamConsumer", "[rtp][consumer]") { 0x80, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0x05, + 0x49, 0x96, 0x02, 0xD2, // SSRC: 1234567890 // Payload (4 bytes). 0xFF, 0xFF, 0xFF, 0xFF, // From here this is just buffer enough for the variable length payload so @@ -244,79 +244,79 @@ SCENARIO("MultiStreamConsumer", "[rtp][consumer]") delete packet; } - // SECTION("RTP packets are not forwarded for unsupported payload types") - // { - // Fixture fixture; + SECTION("RTP packets are not forwarded for unsupported payload types") + { + Fixture fixture; - // // Indicate that the transport is connected in order to activate the consumer. - // dynamic_cast(fixture.consumer.get())->TransportConnected(); + // Indicate that the transport is connected in order to activate the consumer. + dynamic_cast(fixture.consumer.get())->TransportConnected(); - // auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); - // RTC::RTP::SharedPacket sharedPacket(packet); + auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); + RTC::RTP::SharedPacket sharedPacket(packet); - // packet->SetPayloadType(payloadType + 1); + packet->SetPayloadType(payloadType + 1); - // fixture.consumer->SendRtpPacket(packet, sharedPacket); - // fixture.listener->Verify(0); + fixture.consumer->SendRtpPacket(packet, sharedPacket); + fixture.listener->Verify(0); - // delete packet; - // } + delete packet; + } - // SECTION("RTP packets with empty payload are not forwarded") - // { - // Fixture fixture; + SECTION("RTP packets with empty payload are not forwarded") + { + Fixture fixture; - // // Indicate that the transport is connected in order to activate the consumer. - // dynamic_cast(fixture.consumer.get())->TransportConnected(); + // Indicate that the transport is connected in order to activate the consumer. + dynamic_cast(fixture.consumer.get())->TransportConnected(); - // auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 0); - // RTC::RTP::SharedPacket sharedPacket(packet); + auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 0); + RTC::RTP::SharedPacket sharedPacket(packet); - // packet->SetPayloadType(payloadType + 1); + packet->SetPayloadType(payloadType + 1); - // fixture.consumer->SendRtpPacket(packet, sharedPacket); - // fixture.listener->Verify(0); + fixture.consumer->SendRtpPacket(packet, sharedPacket); + fixture.listener->Verify(0); - // delete packet; - // } + delete packet; + } - // SECTION("outgoing RTP packets are forwarded with increased sequence number") - // { - // Fixture fixture; + SECTION("outgoing RTP packets are forwarded with increased sequence number") + { + Fixture fixture; - // // Indicate that the transport is connected in order to activate the consumer. - // dynamic_cast(fixture.consumer.get())->TransportConnected(); + // Indicate that the transport is connected in order to activate the consumer. + dynamic_cast(fixture.consumer.get())->TransportConnected(); - // auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); - // RTC::RTP::SharedPacket sharedPacket(packet); + auto* packet = RTC::RTP::Packet::Parse(buffer, originalPacketLength + 64); + RTC::RTP::SharedPacket sharedPacket(packet); - // uint16_t seq{ 1 }; + uint16_t seq{ 1 }; - // packet->SetSequenceNumber(seq++); - // packet->SetPayloadType(payloadType); - // sharedPacket.Assign(packet); + packet->SetSequenceNumber(seq++); + packet->SetPayloadType(payloadType); + sharedPacket.Assign(packet); - // fixture.consumer->SendRtpPacket(packet, sharedPacket); + fixture.consumer->SendRtpPacket(packet, sharedPacket); - // packet->SetSequenceNumber(seq++); - // sharedPacket.Assign(packet); + packet->SetSequenceNumber(seq++); + sharedPacket.Assign(packet); - // fixture.consumer->SendRtpPacket(packet, sharedPacket); + fixture.consumer->SendRtpPacket(packet, sharedPacket); - // packet->SetSequenceNumber(seq++); - // sharedPacket.Assign(packet); + packet->SetSequenceNumber(seq++); + sharedPacket.Assign(packet); - // fixture.consumer->SendRtpPacket(packet, sharedPacket); + fixture.consumer->SendRtpPacket(packet, sharedPacket); - // packet->SetSequenceNumber(seq++); - // // Remove the payload so it won't be sent. - // packet->RemovePayload(); - // sharedPacket.Assign(packet); + packet->SetSequenceNumber(seq++); + // Remove the payload so it won't be sent. + packet->RemovePayload(); + sharedPacket.Assign(packet); - // fixture.consumer->SendRtpPacket(packet, sharedPacket); + fixture.consumer->SendRtpPacket(packet, sharedPacket); - // fixture.listener->Verify(3); + fixture.listener->Verify(3); - // delete packet; - // } + delete packet; + } } From 15f2012bff6fa4e4b9b8d0e7958bcbee46619ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Mon, 9 Feb 2026 18:37:18 +0100 Subject: [PATCH 03/10] cosmetic --- worker/test/src/RTC/TestMultiStreamConsumer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/worker/test/src/RTC/TestMultiStreamConsumer.cpp b/worker/test/src/RTC/TestMultiStreamConsumer.cpp index 77db0d1339..3fd4cf3012 100644 --- a/worker/test/src/RTC/TestMultiStreamConsumer.cpp +++ b/worker/test/src/RTC/TestMultiStreamConsumer.cpp @@ -182,12 +182,14 @@ namespace SCENARIO("MultiStreamConsumer", "[rtp][consumer]") { + // TODO: We should NOT parse RTP packets for tests anymore. We should use + // RTC::RTP::Packet::Factory() instead. // clang-format off uint8_t buffer[] = { 0x80, 0x01, 0x00, 0x08, 0x00, 0x00, 0x00, 0x04, - 0x49, 0x96, 0x02, 0xD2, // SSRC: 1234567890 + 0x49, 0x96, 0x02, 0xD2, // SSRC: 1234567890 (must be this exact value). // Payload (4 bytes). 0xFF, 0xFF, 0xFF, 0xFF, // From here this is just buffer enough for the variable length payload so From 7f172522bfea24073555eccbbe312490dcea4d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Tue, 10 Feb 2026 15:48:30 +0100 Subject: [PATCH 04/10] more, maybe done --- node/src/test/test-Consumer.ts | 10 +- node/src/test/test-Producer.ts | 105 +- worker/include/RTC/SimpleConsumer.hpp | 103 - worker/include/RTC/SimulcastConsumer.hpp | 141 -- worker/src/RTC/MultiStreamConsumer.cpp | 80 +- worker/src/RTC/SimpleConsumer.cpp | 881 -------- worker/src/RTC/SimulcastConsumer.cpp | 1888 ----------------- .../test/src/RTC/TestMultiStreamConsumer.cpp | 8 +- 8 files changed, 94 insertions(+), 3122 deletions(-) delete mode 100644 worker/include/RTC/SimpleConsumer.hpp delete mode 100644 worker/include/RTC/SimulcastConsumer.hpp delete mode 100644 worker/src/RTC/SimpleConsumer.cpp delete mode 100644 worker/src/RTC/SimulcastConsumer.cpp diff --git a/node/src/test/test-Consumer.ts b/node/src/test/test-Consumer.ts index b486dc6309..89d306d374 100644 --- a/node/src/test/test-Consumer.ts +++ b/node/src/test/test-Consumer.ts @@ -307,7 +307,10 @@ test('transport.consume() succeeds', async () => { producerScore: 0, producerScores: [0], }); - expect(audioConsumer.preferredLayers).toBeUndefined(); + expect(audioConsumer.preferredLayers).toEqual({ + spatialLayer: 0, + temporalLayer: 0, + }); expect(audioConsumer.currentLayers).toBeUndefined(); expect(audioConsumer.appData).toEqual({ baz: 'LOL' }); @@ -879,7 +882,10 @@ test('consumer.setPreferredLayers() succeed', async () => { await audioConsumer.setPreferredLayers({ spatialLayer: 1, temporalLayer: 1 }); - expect(audioConsumer.preferredLayers).toBeUndefined(); + expect(audioConsumer.preferredLayers).toEqual({ + spatialLayer: 0, + temporalLayer: 0, + }); await videoConsumer.setPreferredLayers({ spatialLayer: 2, temporalLayer: 3 }); diff --git a/node/src/test/test-Producer.ts b/node/src/test/test-Producer.ts index a75aa67f36..8689929032 100644 --- a/node/src/test/test-Producer.ts +++ b/node/src/test/test-Producer.ts @@ -14,7 +14,8 @@ import * as FbsProducer from '../fbs/producer'; type TestContext = { mediaCodecs: mediasoup.types.RouterRtpCodecCapability[]; - audioProducerOptions: mediasoup.types.ProducerOptions; + audioProducerOptions1: mediasoup.types.ProducerOptions; + audioProducerOptions2: mediasoup.types.ProducerOptions; videoProducerOptions: mediasoup.types.ProducerOptions; worker?: mediasoup.types.Worker; router?: mediasoup.types.Router; @@ -33,6 +34,11 @@ const ctx: TestContext = { foo: '111', }, }, + { + kind: 'audio', + mimeType: 'audio/PCMA', + clockRate: 8000, + }, { kind: 'video', mimeType: 'video/VP8', @@ -51,7 +57,7 @@ const ctx: TestContext = { rtcpFeedback: [], // Will be ignored. }, ]), - audioProducerOptions: utils.deepFreeze({ + audioProducerOptions1: utils.deepFreeze({ kind: 'audio', rtpParameters: { mid: 'AUDIO', @@ -81,11 +87,34 @@ const ctx: TestContext = { ], // Missing encodings on purpose. rtcp: { - cname: 'audio-1', + cname: 'audio-cname', }, }, appData: { foo: 1, bar: '2' }, }), + audioProducerOptions2: utils.deepFreeze({ + kind: 'audio', + rtpParameters: { + mid: 'AUDIO-2', + codecs: [ + { + mimeType: 'audio/PCMA', + payloadType: 1, + clockRate: 8000, + }, + ], + headerExtensions: [ + { + uri: 'urn:ietf:params:rtp-hdrext:sdes:mid', + id: 10, + }, + ], + encodings: [{ ssrc: 20000000 }], + rtcp: { + cname: 'audio-cname', + }, + }, + }), videoProducerOptions: utils.deepFreeze({ kind: 'video', rtpParameters: { @@ -129,7 +158,7 @@ const ctx: TestContext = { { ssrc: 22222228, rtx: { ssrc: 22222229 } }, ], rtcp: { - cname: 'video-1', + cname: 'video-cname', }, }, appData: { foo: 1, bar: '2' }, @@ -158,33 +187,53 @@ afterEach(async () => { test('webRtcTransport1.produce() succeeds', async () => { const onObserverNewProducer = jest.fn(); - ctx.webRtcTransport1!.observer.once('newproducer', onObserverNewProducer); + ctx.webRtcTransport1!.observer.on('newproducer', onObserverNewProducer); - const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + const audioProducer1 = await ctx.webRtcTransport1!.produce( + ctx.audioProducerOptions1 ); - expect(onObserverNewProducer).toHaveBeenCalledTimes(1); - expect(onObserverNewProducer).toHaveBeenCalledWith(audioProducer); - expect(typeof audioProducer.id).toBe('string'); - expect(audioProducer.closed).toBe(false); - expect(audioProducer.kind).toBe('audio'); - expect(typeof audioProducer.rtpParameters).toBe('object'); - expect(audioProducer.type).toBe('simple'); + const audioProducer2 = await ctx.webRtcTransport1!.produce( + ctx.audioProducerOptions2 + ); + + expect(onObserverNewProducer).toHaveBeenCalledTimes(2); + expect(onObserverNewProducer).toHaveBeenCalledWith(audioProducer1); + expect(onObserverNewProducer).toHaveBeenCalledWith(audioProducer2); + + expect(typeof audioProducer1.id).toBe('string'); + expect(audioProducer1.closed).toBe(false); + expect(audioProducer1.kind).toBe('audio'); + expect(typeof audioProducer1.rtpParameters).toBe('object'); + expect(audioProducer1.type).toBe('simple'); // Private API. - expect(typeof audioProducer.consumableRtpParameters).toBe('object'); - expect(audioProducer.paused).toBe(false); - expect(audioProducer.score).toEqual([]); - expect(audioProducer.appData).toEqual({ foo: 1, bar: '2' }); + expect(typeof audioProducer1.consumableRtpParameters).toBe('object'); + expect(audioProducer1.paused).toBe(false); + expect(audioProducer1.score).toEqual([]); + expect(audioProducer1.appData).toEqual({ foo: 1, bar: '2' }); + + expect(typeof audioProducer2.id).toBe('string'); + expect(audioProducer2.closed).toBe(false); + expect(audioProducer2.kind).toBe('audio'); + expect(typeof audioProducer2.rtpParameters).toBe('object'); + expect(audioProducer2.type).toBe('simple'); + // Private API. + expect(typeof audioProducer2.consumableRtpParameters).toBe('object'); + expect(audioProducer2.paused).toBe(false); + expect(audioProducer2.score).toEqual([]); + expect(audioProducer2.appData).toEqual({}); await expect(ctx.router!.dump()).resolves.toMatchObject({ - mapProducerIdConsumerIds: [{ key: audioProducer.id, values: [] }], + mapProducerIdConsumerIds: expect.arrayContaining([ + { key: audioProducer1.id, values: [] }, + { key: audioProducer2.id, values: [] }, + ]), mapConsumerIdProducerId: [], }); await expect(ctx.webRtcTransport1!.dump()).resolves.toMatchObject({ id: ctx.webRtcTransport1!.id, - producerIds: [audioProducer.id], + producerIds: expect.arrayContaining([audioProducer1.id, audioProducer2.id]), consumerIds: [], }); }, 2000); @@ -489,7 +538,7 @@ test('transport.produce() with no MID and with single encoding without RID or SS test('producer.dump() succeeds', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); const dump1 = await audioProducer.dump(); @@ -599,7 +648,7 @@ test('producer.dump() succeeds', async () => { test('producer.getStats() succeeds', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); const videoProducer = await ctx.webRtcTransport2!.produce( @@ -613,7 +662,7 @@ test('producer.getStats() succeeds', async () => { test('producer.pause() and resume() succeed', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); const onObserverPause = jest.fn(); @@ -647,7 +696,7 @@ test('producer.pause() and resume() succeed', async () => { test('producer.pause() and resume() emit events', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); const promises = []; @@ -672,7 +721,7 @@ test('producer.pause() and resume() emit events', async () => { test('producer.enableTraceEvent() succeed', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); await audioProducer.enableTraceEvent(['rtp', 'pli']); @@ -705,7 +754,7 @@ test('producer.enableTraceEvent() succeed', async () => { test('producer.enableTraceEvent() with wrong arguments rejects with TypeError', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); // @ts-expect-error --- Testing purposes. @@ -776,7 +825,7 @@ test('Producer emits "score"', async () => { test('producer.close() succeeds', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); const onObserverClose = jest.fn(); @@ -801,7 +850,7 @@ test('producer.close() succeeds', async () => { test('Producer methods reject if closed', async () => { const audioProducer = await ctx.webRtcTransport1!.produce( - ctx.audioProducerOptions + ctx.audioProducerOptions1 ); audioProducer.close(); diff --git a/worker/include/RTC/SimpleConsumer.hpp b/worker/include/RTC/SimpleConsumer.hpp deleted file mode 100644 index 0e69241573..0000000000 --- a/worker/include/RTC/SimpleConsumer.hpp +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef MS_RTC_SIMPLE_CONSUMER_HPP -#define MS_RTC_SIMPLE_CONSUMER_HPP - -#include "RTC/Consumer.hpp" -#include "RTC/SeqManager.hpp" -#include "RTC/Shared.hpp" -#include - -namespace RTC -{ - class SimpleConsumer : public RTC::Consumer, public RTC::RTP::RtpStreamSend::Listener - { - public: - SimpleConsumer( - RTC::Shared* shared, - const std::string& id, - const std::string& producerId, - RTC::Consumer::Listener* listener, - const FBS::Transport::ConsumeRequest* data); - ~SimpleConsumer() override; - - public: - flatbuffers::Offset FillBuffer( - flatbuffers::FlatBufferBuilder& builder) const; - flatbuffers::Offset FillBufferStats( - flatbuffers::FlatBufferBuilder& builder) override; - flatbuffers::Offset FillBufferScore( - flatbuffers::FlatBufferBuilder& builder) const override; - bool IsActive() const override - { - // clang-format off - return ( - RTC::Consumer::IsActive() && - this->producerRtpStream && - // If there is no RTP inactivity check do not consider the stream - // inactive despite it has score 0. - (this->producerRtpStream->GetScore() > 0u || !this->producerRtpStream->HasRtpInactivityCheckEnabled()) - ); - // clang-format on - } - void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; - void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; - void ProducerRtpStreamScore( - RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; - void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; - uint8_t GetBitratePriority() const override; - uint32_t IncreaseLayer(uint32_t bitrate, bool considerLoss) override; - void ApplyLayers() override; - uint32_t GetDesiredBitrate() const override; - void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) override; - const std::vector& GetRtpStreams() const override - { - return this->rtpStreams; - } - bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) override; - void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override; - void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) override; - void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) override; - void ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) override; - void ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) override; - uint32_t GetTransmissionRate(uint64_t nowMs) override; - float GetRtt() const override; - - /* Methods inherited from Channel::ChannelSocket::RequestHandler. */ - public: - void HandleRequest(Channel::ChannelRequest* request) override; - - private: - void UserOnTransportConnected() override; - void UserOnTransportDisconnected() override; - void UserOnPaused() override; - void UserOnResumed() override; - void CreateRtpStream(); - void RequestKeyFrame(); - void StorePacketInTargetLayerRetransmissionBuffer( - RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket); - void EmitScore() const; - - /* Pure virtual methods inherited from RtpStreamSend::Listener. */ - public: - void OnRtpStreamScore(RTC::RTP::RtpStream* rtpStream, uint8_t score, uint8_t previousScore) override; - void OnRtpStreamRetransmitRtpPacket( - RTC::RTP::RtpStreamSend* rtpStream, RTC::RTP::Packet* packet) override; - - private: - // Allocated by this. - RTC::RTP::RtpStreamSend* rtpStream{ nullptr }; - // Others. - std::vector rtpStreams; - RTC::RTP::RtpStreamRecv* producerRtpStream{ nullptr }; - bool keyFrameSupported{ false }; - bool syncRequired{ false }; - RTC::SeqManager rtpSeqManager; - bool managingBitrate{ false }; - std::unique_ptr encodingContext; - // Buffer to store packets that arrive earlier than the first packet of the - // video key frame. - std::map::SeqLowerThan> - targetLayerRetransmissionBuffer; - }; -} // namespace RTC - -#endif diff --git a/worker/include/RTC/SimulcastConsumer.hpp b/worker/include/RTC/SimulcastConsumer.hpp deleted file mode 100644 index 6654be5a49..0000000000 --- a/worker/include/RTC/SimulcastConsumer.hpp +++ /dev/null @@ -1,141 +0,0 @@ -#ifndef MS_RTC_SIMULCAST_CONSUMER_HPP -#define MS_RTC_SIMULCAST_CONSUMER_HPP - -#include "RTC/Consumer.hpp" -#include "RTC/RTP/Codecs/PayloadDescriptorHandler.hpp" -#include "RTC/SeqManager.hpp" -#include "RTC/Shared.hpp" -#include - -namespace RTC -{ - class SimulcastConsumer : public RTC::Consumer, public RTC::RTP::RtpStreamSend::Listener - { - public: - SimulcastConsumer( - RTC::Shared* shared, - const std::string& id, - const std::string& producerId, - RTC::Consumer::Listener* listener, - const FBS::Transport::ConsumeRequest* data); - ~SimulcastConsumer() override; - - public: - flatbuffers::Offset FillBuffer( - flatbuffers::FlatBufferBuilder& builder) const; - flatbuffers::Offset FillBufferStats( - flatbuffers::FlatBufferBuilder& builder) override; - flatbuffers::Offset FillBufferScore( - flatbuffers::FlatBufferBuilder& builder) const override; - RTC::ConsumerTypes::VideoLayers GetPreferredLayers() const override - { - RTC::ConsumerTypes::VideoLayers layers; - - layers.spatial = this->preferredLayers.spatial; - layers.temporal = this->preferredLayers.temporal; - - return layers; - } - bool IsActive() const override - { - // clang-format off - return ( - RTC::Consumer::IsActive() && - std::any_of( - this->producerRtpStreams.begin(), - this->producerRtpStreams.end(), - [](const RTC::RTP::RtpStreamRecv* rtpStream) - { - // If there is no RTP inactivity check do not consider the stream - // inactive despite it has score 0. - return (rtpStream != nullptr && (rtpStream->GetScore() > 0u || !rtpStream->HasRtpInactivityCheckEnabled())); - } - ) - ); - // clang-format on - } - void ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; - void ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) override; - void ProducerRtpStreamScore( - RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; - void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; - uint8_t GetBitratePriority() const override; - uint32_t IncreaseLayer(uint32_t bitrate, bool considerLoss) override; - void ApplyLayers() override; - uint32_t GetDesiredBitrate() const override; - void SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) override; - bool GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) override; - const std::vector& GetRtpStreams() const override - { - return this->rtpStreams; - } - void NeedWorstRemoteFractionLost(uint32_t mappedSsrc, uint8_t& worstRemoteFractionLost) override; - void ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) override; - void ReceiveKeyFrameRequest(RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) override; - void ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) override; - void ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) override; - uint32_t GetTransmissionRate(uint64_t nowMs) override; - float GetRtt() const override; - - /* Methods inherited from Channel::ChannelSocket::RequestHandler. */ - public: - void HandleRequest(Channel::ChannelRequest* request) override; - - private: - void UserOnTransportConnected() override; - void UserOnTransportDisconnected() override; - void UserOnPaused() override; - void UserOnResumed() override; - void CreateRtpStream(); - void RequestKeyFrames(); - void RequestKeyFrameForTargetSpatialLayer(); - void RequestKeyFrameForCurrentSpatialLayer(); - void MayChangeLayers(bool force = false); - bool RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const; - void UpdateTargetLayers(int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer); - bool CanSwitchToSpatialLayer(int16_t spatialLayer) const; - void EmitScore() const; - void StorePacketInTargetLayerRetransmissionBuffer( - RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket); - void EmitLayersChange() const; - RTC::RTP::RtpStreamRecv* GetProducerCurrentRtpStream() const; - RTC::RTP::RtpStreamRecv* GetProducerTargetRtpStream() const; - RTC::RTP::RtpStreamRecv* GetProducerTsReferenceRtpStream() const; - - /* Pure virtual methods inherited from RtpStreamSend::Listener. */ - public: - void OnRtpStreamScore(RTC::RTP::RtpStream* rtpStream, uint8_t score, uint8_t previousScore) override; - void OnRtpStreamRetransmitRtpPacket( - RTC::RTP::RtpStreamSend* rtpStream, RTC::RTP::Packet* packet) override; - - private: - // Allocated by this. - RTC::RTP::RtpStreamSend* rtpStream{ nullptr }; - // Others. - absl::flat_hash_map mapMappedSsrcSpatialLayer; - std::vector rtpStreams; - std::vector producerRtpStreams; // Indexed by spatial layer. - bool syncRequired{ false }; - int16_t spatialLayerToSync{ -1 }; - bool lastSentPacketHasMarker{ false }; - RTC::SeqManager rtpSeqManager; - RTC::ConsumerTypes::VideoLayers preferredLayers; - RTC::ConsumerTypes::VideoLayers provisionalTargetLayers; - RTC::ConsumerTypes::VideoLayers targetLayers; - int16_t currentSpatialLayer{ -1 }; - int16_t tsReferenceSpatialLayer{ -1 }; // Used for RTP TS sync. - uint16_t snReferenceSpatialLayer{ 0 }; - bool checkingForOldPacketsInSpatialLayer{ false }; - std::unique_ptr encodingContext; - uint32_t tsOffset{ 0u }; // RTP Timestamp offset. - bool keyFrameForTsOffsetRequested{ false }; - // Last time we moved to lower spatial layer due to BWE. - uint64_t lastBweDowngradeAtMs{ 0u }; - // Buffer to store packets that arrive earlier than the first packet of the - // video key frame. - std::map::SeqLowerThan> - targetLayerRetransmissionBuffer; - }; -} // namespace RTC - -#endif diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp index 6b45fe7022..f1d36d99e9 100644 --- a/worker/src/RTC/MultiStreamConsumer.cpp +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -1,6 +1,5 @@ #define MS_CLASS "RTC::MultiStreamConsumer" -// TODO: COmment -#define MS_LOG_DEV_LEVEL 3 +// #define MS_LOG_DEV_LEVEL 3 #include "RTC/MultiStreamConsumer.hpp" #include "DepLibUV.hpp" @@ -66,9 +65,6 @@ namespace RTC { auto& encoding = this->consumableRtpEncodings[idx]; - // TODO - MS_DUMP("------ encoding.ssrc:%" PRIu32, encoding.ssrc); - this->mapMappedSsrcSpatialLayer[encoding.ssrc] = static_cast(idx); } @@ -327,9 +323,6 @@ namespace RTC { MS_TRACE(); - // TODO - MS_DUMP("------ mappedSsrc:%" PRIu32, mappedSsrc); - auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); @@ -343,9 +336,6 @@ namespace RTC { MS_TRACE(); - // TODO - MS_DUMP("------ mappedSsrc:%" PRIu32, mappedSsrc); - auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); @@ -513,12 +503,6 @@ namespace RTC goto done; } - // TODO - MS_DUMP( - "---- 1 spatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", - spatialLayer, - this->producerRtpStreams.size()); - // This can be null. auto* producerRtpStream = this->producerRtpStreams.at(spatialLayer); @@ -541,12 +525,6 @@ namespace RTC this->provisionalTargetLayers.spatial != -1 && producerRtpStream->GetActiveMs() < StreamMinActiveMs) { - // TODO - MS_DUMP( - "---- 2 this->provisionalTargetLayers.spatial:%" PRIi16 ", producerRtpStreams.size():%zu", - this->provisionalTargetLayers.spatial, - this->producerRtpStreams.size()); - const auto* provisionalProducerRtpStream = this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); @@ -587,12 +565,6 @@ namespace RTC requiredBitrate && temporalLayer == 0 && this->provisionalTargetLayers.spatial > -1 && spatialLayer > this->provisionalTargetLayers.spatial) { - // TODO - MS_DUMP( - "---- 3 this->provisionalTargetLayers.spatial:%" PRIi16 ", producerRtpStreams.size():%zu", - this->provisionalTargetLayers.spatial, - this->producerRtpStreams.size()); - auto* provisionalProducerRtpStream = this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); auto provisionalRequiredBitrate = provisionalProducerRtpStream->GetBitrate( @@ -737,12 +709,6 @@ namespace RTC // iterate all streams here anyway. for (auto sIdx{ static_cast(this->producerRtpStreams.size() - 1) }; sIdx >= 0; --sIdx) { - // TODO - MS_DUMP( - "---- 4 sIdx:%" PRIi16 ", producerRtpStreams.size():%zu", - sIdx, - this->producerRtpStreams.size()); - auto* producerRtpStream = this->producerRtpStreams.at(sIdx); if (!producerRtpStream) @@ -773,20 +739,6 @@ namespace RTC packet->logger.consumerId = this->id; #endif - // TODO - MS_DUMP( - "---- 5 packet->GetSsrc():%" PRIu32 ", mapMappedSsrcSpatialLayer.size():%zu", - packet->GetSsrc(), - this->mapMappedSsrcSpatialLayer.size()); - - // TODO - MS_DUMP("------ 5.1 mapMappedSsrcSpatialLayer>"); - for (const auto& kv : this->mapMappedSsrcSpatialLayer) - { - MS_DUMP("------------ mappeSsrc:%" PRIu32 ", spatialLayer:%" PRIi16, kv.first, kv.second); - } - MS_DUMP("------ 5.1 mapMappedSsrcSpatialLayer>"); - auto spatialLayer = this->mapMappedSsrcSpatialLayer.at(packet->GetSsrc()); if (!IsActive()) @@ -1672,11 +1624,7 @@ namespace RTC for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx) { - auto spatialLayer = static_cast(sIdx); - - // TODO - MS_DUMP("---- 6 sIdx:%zu, producerRtpStreams.size():%zu", sIdx, this->producerRtpStreams.size()); - + auto spatialLayer = static_cast(sIdx); auto* producerRtpStream = this->producerRtpStreams.at(sIdx); auto producerScore = producerRtpStream ? producerRtpStream->GetScore() : 0u; @@ -1813,12 +1761,6 @@ namespace RTC { MS_TRACE(); - // TODO - MS_DUMP( - "---- 7 spatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", - spatialLayer, - this->producerRtpStreams.size()); - // This method assumes that the caller has verified that there is a valid // Producer RtpStream for the given spatial layer. MS_ASSERT( @@ -1924,12 +1866,6 @@ namespace RTC return nullptr; } - // TODO - MS_DUMP( - "---- 8 this->currentSpatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", - this->currentSpatialLayer, - this->producerRtpStreams.size()); - // This may return nullptr. return this->producerRtpStreams.at(this->currentSpatialLayer); } @@ -1943,12 +1879,6 @@ namespace RTC return nullptr; } - // TODO - MS_DUMP( - "---- 9 this->targetLayers.spatial:%" PRIi16 ", producerRtpStreams.size():%zu", - this->targetLayers.spatial, - this->producerRtpStreams.size()); - // This may return nullptr. return this->producerRtpStreams.at(this->targetLayers.spatial); } @@ -1962,12 +1892,6 @@ namespace RTC return nullptr; } - // TODO - MS_DUMP( - "---- 10 this->tsReferenceSpatialLayer:%" PRIi16 ", producerRtpStreams.size():%zu", - this->tsReferenceSpatialLayer, - this->producerRtpStreams.size()); - // This may return nullptr. return this->producerRtpStreams.at(this->tsReferenceSpatialLayer); } diff --git a/worker/src/RTC/SimpleConsumer.cpp b/worker/src/RTC/SimpleConsumer.cpp deleted file mode 100644 index 4f710136e8..0000000000 --- a/worker/src/RTC/SimpleConsumer.cpp +++ /dev/null @@ -1,881 +0,0 @@ -#define MS_CLASS "RTC::SimpleConsumer" -// #define MS_LOG_DEV_LEVEL 3 - -#include "RTC/SimpleConsumer.hpp" -#include "DepLibUV.hpp" -#include "Logger.hpp" -#include "MediaSoupErrors.hpp" -#include "Utils.hpp" -#include "RTC/RTP/Codecs/Tools.hpp" -#ifdef MS_RTC_LOGGER_RTP -#include "RTC/RtcLogger.hpp" -#endif -#include // std::numeric_limits - -namespace RTC -{ - /* Static. */ - - static constexpr size_t TargetLayerRetransmissionBufferSize{ 15u }; - - /* Instance methods. */ - - SimpleConsumer::SimpleConsumer( - RTC::Shared* shared, - const std::string& id, - const std::string& producerId, - RTC::Consumer::Listener* listener, - const FBS::Transport::ConsumeRequest* data) - : RTC::Consumer::Consumer(shared, id, producerId, listener, data, RTC::RtpParameters::Type::SIMPLE) - { - MS_TRACE(); - - // Ensure there is a single encoding. - if (this->consumableRtpEncodings.size() != 1u) - { - MS_THROW_TYPE_ERROR("invalid consumableRtpEncodings with size != 1"); - } - - auto& encoding = this->rtpParameters.encodings[0]; - const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); - - this->keyFrameSupported = RTC::RTP::Codecs::Tools::CanBeKeyFrame(mediaCodec->mimeType); - - // Create RtpStreamSend instance for sending a single stream to the remote. - CreateRtpStream(); - - // Let's chosee an initial output seq number between 1000 and 32768 to avoid - // libsrtp bug: - // https://github.com/versatica/mediasoup/issues/1437 - const uint16_t initialOutputSeq = - Utils::Crypto::GetRandomUInt(1000u, std::numeric_limits::max() / 2); - - this->rtpSeqManager = RTC::SeqManager(initialOutputSeq); - - // Create the encoding context for Opus. - if ( - mediaCodec->mimeType.type == RTC::RtpCodecMimeType::Type::AUDIO && - (mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::OPUS || - mediaCodec->mimeType.subtype == RTC::RtpCodecMimeType::Subtype::MULTIOPUS)) - { - RTC::RTP::Codecs::EncodingContext::Params params; - - this->encodingContext.reset( - RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params)); - - // ignoreDtx is set to false by default. - this->encodingContext->SetIgnoreDtx(data->ignoreDtx()); - } - - // NOTE: This may throw. - this->shared->channelMessageRegistrator->RegisterHandler( - this->id, - /*channelRequestHandler*/ this, - /*channelNotificationHandler*/ nullptr); - } - - SimpleConsumer::~SimpleConsumer() - { - MS_TRACE(); - - this->shared->channelMessageRegistrator->UnregisterHandler(this->id); - - delete this->rtpStream; - this->targetLayerRetransmissionBuffer.clear(); - } - - flatbuffers::Offset SimpleConsumer::FillBuffer( - flatbuffers::FlatBufferBuilder& builder) const - { - MS_TRACE(); - - // Call the parent method. - auto base = RTC::Consumer::FillBuffer(builder); - // Add rtpStream. - std::vector> rtpStreams; - rtpStreams.emplace_back(this->rtpStream->FillBuffer(builder)); - - auto dump = FBS::Consumer::CreateConsumerDumpDirect(builder, base, &rtpStreams); - - return FBS::Consumer::CreateDumpResponse(builder, dump); - } - - flatbuffers::Offset SimpleConsumer::FillBufferStats( - flatbuffers::FlatBufferBuilder& builder) - { - MS_TRACE(); - - std::vector> rtpStreams; - - // Add stats of our send stream. - rtpStreams.emplace_back(this->rtpStream->FillBufferStats(builder)); - - // Add stats of our recv stream. - if (this->producerRtpStream) - { - rtpStreams.emplace_back(this->producerRtpStream->FillBufferStats(builder)); - } - - return FBS::Consumer::CreateGetStatsResponseDirect(builder, &rtpStreams); - } - - flatbuffers::Offset SimpleConsumer::FillBufferScore( - flatbuffers::FlatBufferBuilder& builder) const - { - MS_TRACE(); - - MS_ASSERT(this->producerRtpStreamScores, "producerRtpStreamScores not set"); - - uint8_t producerScore{ 0 }; - - if (this->producerRtpStream) - { - producerScore = this->producerRtpStream->GetScore(); - } - - return FBS::Consumer::CreateConsumerScoreDirect( - builder, this->rtpStream->GetScore(), producerScore, this->producerRtpStreamScores); - } - - void SimpleConsumer::HandleRequest(Channel::ChannelRequest* request) - { - MS_TRACE(); - - switch (request->method) - { - case Channel::ChannelRequest::Method::CONSUMER_DUMP: - { - auto dumpOffset = FillBuffer(request->GetBufferBuilder()); - - request->Accept(FBS::Response::Body::Consumer_DumpResponse, dumpOffset); - - break; - } - - case Channel::ChannelRequest::Method::CONSUMER_REQUEST_KEY_FRAME: - { - if (IsActive()) - { - RequestKeyFrame(); - } - - request->Accept(); - - break; - } - - case Channel::ChannelRequest::Method::CONSUMER_SET_PREFERRED_LAYERS: - { - // Accept with empty preferred layers object. - - auto responseOffset = - FBS::Consumer::CreateSetPreferredLayersResponse(request->GetBufferBuilder()); - - request->Accept(FBS::Response::Body::Consumer_SetPreferredLayersResponse, responseOffset); - - break; - } - - default: - { - // Pass it to the parent class. - RTC::Consumer::HandleRequest(request); - } - } - } - - void SimpleConsumer::ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t /*mappedSsrc*/) - { - MS_TRACE(); - - this->producerRtpStream = rtpStream; - } - - void SimpleConsumer::ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t /*mappedSsrc*/) - { - MS_TRACE(); - - this->producerRtpStream = rtpStream; - - // Emit the score event. - EmitScore(); - } - - void SimpleConsumer::ProducerRtpStreamScore( - RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) - { - MS_TRACE(); - - // Emit the score event. - EmitScore(); - } - - void SimpleConsumer::ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* /*rtpStream*/, bool /*first*/) - { - MS_TRACE(); - - // Do nothing. - } - - uint8_t SimpleConsumer::GetBitratePriority() const - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - - // Audio SimpleConsumer does not play the BWE game. - if (this->kind != RTC::Media::Kind::VIDEO) - { - return 0u; - } - - if (!IsActive()) - { - return 0u; - } - - return this->priority; - } - - uint32_t SimpleConsumer::IncreaseLayer(uint32_t bitrate, bool /*considerLoss*/) - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - MS_ASSERT(this->kind == RTC::Media::Kind::VIDEO, "should be video"); - MS_ASSERT(IsActive(), "should be active"); - - // If this is not the first time this method is called within the same iteration, - // return 0 since a video SimpleConsumer does not keep state about this. - if (this->managingBitrate) - { - return 0u; - } - - this->managingBitrate = true; - - // Video SimpleConsumer does not really play the BWE game when. However, let's - // be honest and try to be nice. - auto nowMs = DepLibUV::GetTimeMs(); - auto desiredBitrate = this->producerRtpStream->GetBitrate(nowMs); - - if (desiredBitrate < bitrate) - { - return desiredBitrate; - } - else - { - return bitrate; - } - } - - void SimpleConsumer::ApplyLayers() - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - MS_ASSERT(this->kind == RTC::Media::Kind::VIDEO, "should be video"); - MS_ASSERT(IsActive(), "should be active"); - - this->managingBitrate = false; - - // SimpleConsumer does not play the BWE game (even if video kind). - } - - uint32_t SimpleConsumer::GetDesiredBitrate() const - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - - // Audio SimpleConsumer does not play the BWE game. - if (this->kind != RTC::Media::Kind::VIDEO) - { - return 0u; - } - - if (!IsActive()) - { - return 0u; - } - - auto nowMs = DepLibUV::GetTimeMs(); - auto desiredBitrate = this->producerRtpStream->GetBitrate(nowMs); - - // If consumer.rtpParameters.encodings[0].maxBitrate was given and it's - // greater than computed one, then use it. - auto maxBitrate = this->rtpParameters.encodings[0].maxBitrate; - - desiredBitrate = std::max(maxBitrate, desiredBitrate); - - return desiredBitrate; - } - - // NOLINTNEXTLINE(misc-no-recursion) - void SimpleConsumer::SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) - { - MS_TRACE(); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.consumerId = this->id; -#endif - - if (!IsActive()) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::CONSUMER_INACTIVE); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - - return; - } - - // If we need to sync, support key frames and this is not a key frame, ignore - // the packet. - if (this->syncRequired && this->keyFrameSupported && !packet->IsKeyFrame()) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); -#endif - - // NOTE: No need to drop the packet in the RTP sequence manager since here - // we are blocking all packets but the key frame that would trigger sync - // below. - - // Store the packet for the scenario in which this packet is part of the - // key frame and it arrived before the first packet of the key frame. - StorePacketInTargetLayerRetransmissionBuffer(packet, sharedPacket); - - return; - } - - auto payloadType = packet->GetPayloadType(); - - // NOTE: This may happen if this Consumer supports just some codecs of those - // in the corresponding Producer. - if (!this->supportedCodecPayloadTypes[payloadType]) - { - MS_WARN_DEV("payload type not supported [payloadType:%" PRIu8 "]", payloadType); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::UNSUPPORTED_PAYLOAD_TYPE); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - - return; - } - - // Packets with only padding are not forwarded. - if (packet->GetPayloadLength() == 0) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - - return; - } - - bool marker; - - // Process the payload if needed. Drop packet if necessary. - if (this->encodingContext && !packet->ProcessPayload(this->encodingContext.get(), marker)) - { - MS_DEBUG_DEV( - "discarding packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::DROPPED_BY_CODEC); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - - return; - } - - // Whether this is the first packet after re-sync. - const bool isSyncPacket = this->syncRequired; - - // Whether packets stored in the target layer retransmission buffer must be - // sent once this packet is sent. - bool sendPacketsInTargetLayerRetransmissionBuffer{ false }; - - // Sync sequence number and timestamp if required. - if (isSyncPacket) - { - if (packet->IsKeyFrame()) - { - MS_DEBUG_TAG( - rtp, - "sync key frame received [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - - sendPacketsInTargetLayerRetransmissionBuffer = true; - } - - this->rtpSeqManager.Sync(packet->GetSequenceNumber() - 1); - - this->syncRequired = false; - } - - // Update RTP seq number and timestamp. - uint16_t seq; - - this->rtpSeqManager.Input(packet->GetSequenceNumber(), seq); - - // Save original packet fields. - auto origSsrc = packet->GetSsrc(); - auto origSeq = packet->GetSequenceNumber(); - - // Rewrite packet. - packet->SetSsrc(this->rtpParameters.encodings[0].ssrc); - packet->SetSequenceNumber(seq); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.sendRtpTimestamp = packet->GetTimestamp(); - packet->logger.sendSeqNumber = seq; -#endif - - if (isSyncPacket) - { - MS_DEBUG_TAG( - rtp, - "sending sync packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 - "] from original [seq:%" PRIu16 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp(), - origSeq); - } - - const RTC::RTP::RtpStreamSend::ReceivePacketResult result = - this->rtpStream->ReceivePacket(packet, sharedPacket); - - if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) - { - // Send the packet. - this->listener->OnConsumerSendRtpPacket(this, packet); - - // May emit 'trace' event. - EmitTraceEventRtpAndKeyFrameTypes(packet); - } - else - { - MS_WARN_TAG( - rtp, - "failed to send packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 - "] from original [seq:%" PRIu16 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp(), - origSeq); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::SEND_RTP_STREAM_DISCARDED); -#endif - } - - // Restore packet fields. - packet->SetSsrc(origSsrc); - packet->SetSequenceNumber(origSeq); - - // If sharedPacket doesn't have a packet inside and it has been stored we - // need to clone the packet into it. - if (!sharedPacket.HasPacket() && result == RTC::RTP::RtpStreamSend::ReceivePacketResult::ACCEPTED_AND_STORED) - { - sharedPacket.Assign(packet); - } - - // If sent packet was the first packet of a key frame, let's send buffered - // packets belonging to the same key frame that arrived earlier due to - // packet misorder. - if (sendPacketsInTargetLayerRetransmissionBuffer) - { - // NOTE: Only send buffered packets if the first packet containing the key - // frame was sent. - if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) - { - for (auto& kv : this->targetLayerRetransmissionBuffer) - { - auto& bufferedSharedPacket = kv.second; - auto* bufferedPacket = bufferedSharedPacket.GetPacket(); - - if (bufferedPacket->GetSequenceNumber() > origSeq) - { - MS_DEBUG_DEV( - "sending packet buffered in the target layer retransmission buffer [ssrc:%" PRIu32 - ", seq:%" PRIu16 ", ts:%" PRIu32 - "] after sending first packet of the key frame [ssrc:%" PRIu32 ", seq:%" PRIu16 - ", ts:%" PRIu32 "]", - bufferedPacket->GetSsrc(), - bufferedPacket->GetSequenceNumber(), - bufferedPacket->GetTimestamp(), - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - - SendRtpPacket(bufferedPacket, bufferedSharedPacket); - - // Be sure that the target layer retransmission buffer has not been - // emptied as a result of sending this packet. If so, exit the loop. - if (this->targetLayerRetransmissionBuffer.empty()) - { - MS_DEBUG_DEV( - "target layer retransmission buffer emptied while iterating it, exiting the loop"); - - break; - } - } - } - } - - this->targetLayerRetransmissionBuffer.clear(); - } - } - - bool SimpleConsumer::GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) - { - MS_TRACE(); - - if (static_cast((nowMs - this->lastRtcpSentTime) * 1.15) < this->maxRtcpInterval) - { - return true; - } - - auto* senderReport = this->rtpStream->GetRtcpSenderReport(nowMs); - - if (!senderReport) - { - return true; - } - - // Build SDES chunk for this sender. - auto* sdesChunk = this->rtpStream->GetRtcpSdesChunk(); - - auto* delaySinceLastRrSsrcInfo = this->rtpStream->GetRtcpXrDelaySinceLastRrSsrcInfo(nowMs); - - // RTCP Compound packet buffer cannot hold the data. - if (!packet->Add(senderReport, sdesChunk, delaySinceLastRrSsrcInfo)) - { - return false; - } - - this->lastRtcpSentTime = nowMs; - - return true; - } - - void SimpleConsumer::NeedWorstRemoteFractionLost( - uint32_t /*mappedSsrc*/, uint8_t& worstRemoteFractionLost) - { - MS_TRACE(); - - if (!IsActive()) - { - return; - } - - auto fractionLost = this->rtpStream->GetFractionLost(); - - // If our fraction lost is worse than the given one, update it. - worstRemoteFractionLost = std::max(fractionLost, worstRemoteFractionLost); - } - - void SimpleConsumer::ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) - { - MS_TRACE(); - - if (!IsActive()) - { - return; - } - - // May emit 'trace' event. - EmitTraceEventNackType(); - - this->rtpStream->ReceiveNack(nackPacket); - } - - void SimpleConsumer::ReceiveKeyFrameRequest( - RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) - { - MS_TRACE(); - - switch (messageType) - { - case RTC::RTCP::FeedbackPs::MessageType::PLI: - { - EmitTraceEventPliType(ssrc); - - break; - } - - case RTC::RTCP::FeedbackPs::MessageType::FIR: - { - EmitTraceEventFirType(ssrc); - - break; - } - - default:; - } - - this->rtpStream->ReceiveKeyFrameRequest(messageType); - - if (IsActive()) - { - RequestKeyFrame(); - } - } - - void SimpleConsumer::ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) - { - MS_TRACE(); - - this->rtpStream->ReceiveRtcpReceiverReport(report); - } - - void SimpleConsumer::ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) - { - MS_TRACE(); - - this->rtpStream->ReceiveRtcpXrReceiverReferenceTime(report); - } - - uint32_t SimpleConsumer::GetTransmissionRate(uint64_t nowMs) - { - MS_TRACE(); - - if (!IsActive()) - { - return 0u; - } - - return this->rtpStream->GetBitrate(nowMs); - } - - float SimpleConsumer::GetRtt() const - { - MS_TRACE(); - - return this->rtpStream->GetRtt(); - } - - void SimpleConsumer::UserOnTransportConnected() - { - MS_TRACE(); - - this->syncRequired = true; - - if (IsActive()) - { - RequestKeyFrame(); - } - } - - void SimpleConsumer::UserOnTransportDisconnected() - { - MS_TRACE(); - - this->rtpStream->Pause(); - this->targetLayerRetransmissionBuffer.clear(); - } - - void SimpleConsumer::UserOnPaused() - { - MS_TRACE(); - - this->rtpStream->Pause(); - this->targetLayerRetransmissionBuffer.clear(); - - if (this->externallyManagedBitrate && this->kind == RTC::Media::Kind::VIDEO) - { - this->listener->OnConsumerNeedZeroBitrate(this); - } - } - - void SimpleConsumer::UserOnResumed() - { - MS_TRACE(); - - this->syncRequired = true; - - if (IsActive()) - { - RequestKeyFrame(); - } - } - - void SimpleConsumer::CreateRtpStream() - { - MS_TRACE(); - - auto& encoding = this->rtpParameters.encodings[0]; - const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); - - MS_DEBUG_TAG( - rtp, "[ssrc:%" PRIu32 ", payloadType:%" PRIu8 "]", encoding.ssrc, mediaCodec->payloadType); - - // Set stream params. - RTC::RTP::RtpStream::Params params; - - params.ssrc = encoding.ssrc; - params.payloadType = mediaCodec->payloadType; - params.mimeType = mediaCodec->mimeType; - params.clockRate = mediaCodec->clockRate; - params.cname = this->rtpParameters.rtcp.cname; - - // Check in band FEC in codec parameters. - if (mediaCodec->parameters.HasInteger("useinbandfec") && mediaCodec->parameters.GetInteger("useinbandfec") == 1) - { - MS_DEBUG_TAG(rtp, "in band FEC enabled"); - - params.useInBandFec = true; - } - - // Check DTX in codec parameters. - if (mediaCodec->parameters.HasInteger("usedtx") && mediaCodec->parameters.GetInteger("usedtx") == 1) - { - MS_DEBUG_TAG(rtp, "DTX enabled"); - - params.useDtx = true; - } - - // Check DTX in the encoding. - if (encoding.dtx) - { - MS_DEBUG_TAG(rtp, "DTX enabled"); - - params.useDtx = true; - } - - for (const auto& fb : mediaCodec->rtcpFeedback) - { - if (!params.useNack && fb.type == "nack" && fb.parameter.empty()) - { - MS_DEBUG_2TAGS(rtp, rtcp, "NACK supported"); - - params.useNack = true; - } - else if (!params.usePli && fb.type == "nack" && fb.parameter == "pli") - { - MS_DEBUG_2TAGS(rtp, rtcp, "PLI supported"); - - params.usePli = true; - } - else if (!params.useFir && fb.type == "ccm" && fb.parameter == "fir") - { - MS_DEBUG_2TAGS(rtp, rtcp, "FIR supported"); - - params.useFir = true; - } - } - - this->rtpStream = new RTC::RTP::RtpStreamSend(this, params, this->rtpParameters.mid); - this->rtpStreams.push_back(this->rtpStream); - - // If the Consumer is paused, tell the RtpStreamSend. - if (IsPaused() || IsProducerPaused()) - { - this->rtpStream->Pause(); - } - - const auto* rtxCodec = this->rtpParameters.GetRtxCodecForEncoding(encoding); - - if (rtxCodec && encoding.hasRtx) - { - this->rtpStream->SetRtx(rtxCodec->payloadType, encoding.rtx.ssrc); - } - } - - void SimpleConsumer::RequestKeyFrame() - { - MS_TRACE(); - - if (this->kind != RTC::Media::Kind::VIDEO) - { - return; - } - - auto mappedSsrc = this->consumableRtpEncodings[0].ssrc; - - this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); - } - - void SimpleConsumer::StorePacketInTargetLayerRetransmissionBuffer( - RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) - { - MS_TRACE(); - - MS_DEBUG_DEV( - "storing packet in target layer retransmission buffer [ssrc:%" PRIu32 ", seq:%" PRIu16 - ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - - // Store original packet into the buffer. Only clone once and only if - // necessary. - if (!sharedPacket.HasPacket()) - { - sharedPacket.Assign(packet); - } - // Assert that, if sharedPacket was already filled, both packet and - // sharedPacket are the very same RTP packet. - else - { - sharedPacket.AssertSamePacket(packet); - } - - this->targetLayerRetransmissionBuffer[packet->GetSequenceNumber()] = sharedPacket; - - if (this->targetLayerRetransmissionBuffer.size() > TargetLayerRetransmissionBufferSize) - { - this->targetLayerRetransmissionBuffer.erase(this->targetLayerRetransmissionBuffer.begin()); - } - } - - void SimpleConsumer::EmitScore() const - { - MS_TRACE(); - - auto scoreOffset = FillBufferScore(this->shared->channelNotifier->GetBufferBuilder()); - - auto notificationOffset = FBS::Consumer::CreateScoreNotification( - this->shared->channelNotifier->GetBufferBuilder(), scoreOffset); - - this->shared->channelNotifier->Emit( - this->id, - FBS::Notification::Event::CONSUMER_SCORE, - FBS::Notification::Body::Consumer_ScoreNotification, - notificationOffset); - } - - void SimpleConsumer::OnRtpStreamScore( - RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) - { - MS_TRACE(); - - // Emit the score event. - EmitScore(); - } - - void SimpleConsumer::OnRtpStreamRetransmitRtpPacket( - RTC::RTP::RtpStreamSend* /*rtpStream*/, RTC::RTP::Packet* packet) - { - MS_TRACE(); - - this->listener->OnConsumerRetransmitRtpPacket(this, packet); - - // May emit 'trace' event. - EmitTraceEventRtpAndKeyFrameTypes(packet, this->rtpStream->HasRtx()); - } -} // namespace RTC diff --git a/worker/src/RTC/SimulcastConsumer.cpp b/worker/src/RTC/SimulcastConsumer.cpp deleted file mode 100644 index ccde25bd08..0000000000 --- a/worker/src/RTC/SimulcastConsumer.cpp +++ /dev/null @@ -1,1888 +0,0 @@ -#define MS_CLASS "RTC::SimulcastConsumer" -// #define MS_LOG_DEV_LEVEL 3 - -#include "RTC/SimulcastConsumer.hpp" -#include "DepLibUV.hpp" -#include "Logger.hpp" -#include "MediaSoupErrors.hpp" -#include "Utils.hpp" -#include "RTC/RTP/Codecs/Tools.hpp" -#ifdef MS_RTC_LOGGER_RTP -#include "RTC/RtcLogger.hpp" -#endif -#include // std::numeric_limits - -namespace RTC -{ - /* Static. */ - - static constexpr uint64_t StreamMinActiveMs{ 2000u }; - static constexpr uint64_t BweDowngradeConservativeMs{ 10000u }; - static constexpr uint64_t BweDowngradeMinActiveMs{ 8000u }; - static constexpr uint16_t MaxSequenceNumberGap{ 100u }; - static constexpr size_t TargetLayerRetransmissionBufferSize{ 30u }; - - /* Instance methods. */ - - SimulcastConsumer::SimulcastConsumer( - RTC::Shared* shared, - const std::string& id, - const std::string& producerId, - RTC::Consumer::Listener* listener, - const FBS::Transport::ConsumeRequest* data) - : RTC::Consumer::Consumer( - shared, id, producerId, listener, data, RTC::RtpParameters::Type::SIMULCAST) - { - MS_TRACE(); - - // We allow a single encoding in simulcast (so we can enable temporal layers - // with a single simulcast stream). - // NOTE: No need to check this->consumableRtpEncodings.size() > 0 here since - // it's already done in Consumer constructor. - - auto& encoding = this->rtpParameters.encodings[0]; - - // Ensure there are as many spatial layers as encodings. - if (encoding.spatialLayers != this->consumableRtpEncodings.size()) - { - MS_THROW_TYPE_ERROR("encoding.spatialLayers does not match number of consumableRtpEncodings"); - } - - // Fill mapMappedSsrcSpatialLayer. - for (size_t idx{ 0u }; idx < this->consumableRtpEncodings.size(); ++idx) - { - auto& encoding = this->consumableRtpEncodings[idx]; - - this->mapMappedSsrcSpatialLayer[encoding.ssrc] = static_cast(idx); - } - - // Set preferredLayers (if given). - if (flatbuffers::IsFieldPresent(data, FBS::Transport::ConsumeRequest::VT_PREFERREDLAYERS)) - { - const auto* preferredLayers = data->preferredLayers(); - - this->preferredLayers.spatial = preferredLayers->spatialLayer(); - - if (this->preferredLayers.spatial > encoding.spatialLayers - 1) - { - this->preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); - } - - if (auto preferredTemporalLayer = preferredLayers->temporalLayer(); - preferredTemporalLayer.has_value()) - { - this->preferredLayers.temporal = preferredTemporalLayer.value(); - - if (this->preferredLayers.temporal > encoding.temporalLayers - 1) - { - this->preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); - } - } - else - { - this->preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); - } - } - else - { - // Initially set preferredSpatialLayer and preferredTemporalLayer to the - // maximum value. - this->preferredLayers.spatial = static_cast(encoding.spatialLayers - 1); - this->preferredLayers.temporal = static_cast(encoding.temporalLayers - 1); - } - - // Reserve space for the Producer RTP streams by filling all the possible - // entries with nullptr. - this->producerRtpStreams.insert( - this->producerRtpStreams.begin(), this->consumableRtpEncodings.size(), nullptr); - - // Create the encoding context. - const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); - - if (!RTC::RTP::Codecs::Tools::IsValidTypeForCodec(this->type, mediaCodec->mimeType)) - { - MS_THROW_TYPE_ERROR( - "%s codec not supported for simulcast", mediaCodec->mimeType.ToString().c_str()); - } - - // Let's chosee an initial output seq number between 1000 and 32768 to avoid - // libsrtp bug: - // https://github.com/versatica/mediasoup/issues/1437 - const uint16_t initialOutputSeq = - Utils::Crypto::GetRandomUInt(1000u, std::numeric_limits::max() / 2); - - this->rtpSeqManager = RTC::SeqManager(initialOutputSeq); - - RTC::RTP::Codecs::EncodingContext::Params params; - - params.spatialLayers = encoding.spatialLayers; - params.temporalLayers = encoding.temporalLayers; - - this->encodingContext.reset( - RTC::RTP::Codecs::Tools::GetEncodingContext(mediaCodec->mimeType, params)); - - MS_ASSERT(this->encodingContext, "no encoding context for this codec"); - - // Create RtpStreamSend instance for sending a single stream to the remote. - CreateRtpStream(); - - // NOTE: This may throw. - this->shared->channelMessageRegistrator->RegisterHandler( - this->id, - /*channelRequestHandler*/ this, - /*channelRequestHandler*/ nullptr); - } - - SimulcastConsumer::~SimulcastConsumer() - { - MS_TRACE(); - - this->shared->channelMessageRegistrator->UnregisterHandler(this->id); - - delete this->rtpStream; - this->targetLayerRetransmissionBuffer.clear(); - } - - flatbuffers::Offset SimulcastConsumer::FillBuffer( - flatbuffers::FlatBufferBuilder& builder) const - { - MS_TRACE(); - - // Call the parent method. - auto base = RTC::Consumer::FillBuffer(builder); - // Add rtpStream. - std::vector> rtpStreams; - rtpStreams.emplace_back(this->rtpStream->FillBuffer(builder)); - - auto dump = FBS::Consumer::CreateConsumerDumpDirect( - builder, - base, - &rtpStreams, - this->preferredLayers.spatial, - this->targetLayers.spatial, - this->currentSpatialLayer, - this->preferredLayers.temporal, - this->targetLayers.temporal, - this->encodingContext->GetCurrentTemporalLayer()); - - return FBS::Consumer::CreateDumpResponse(builder, dump); - } - - flatbuffers::Offset SimulcastConsumer::FillBufferStats( - flatbuffers::FlatBufferBuilder& builder) - { - MS_TRACE(); - - std::vector> rtpStreams; - - // Add stats of our send stream. - rtpStreams.emplace_back(this->rtpStream->FillBufferStats(builder)); - - auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); - - // Add stats of our recv stream. - if (producerCurrentRtpStream) - { - rtpStreams.emplace_back(producerCurrentRtpStream->FillBufferStats(builder)); - } - - return FBS::Consumer::CreateGetStatsResponseDirect(builder, &rtpStreams); - } - - flatbuffers::Offset SimulcastConsumer::FillBufferScore( - flatbuffers::FlatBufferBuilder& builder) const - { - MS_TRACE(); - - MS_ASSERT(this->producerRtpStreamScores, "producerRtpStreamScores not set"); - - auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); - - uint8_t producerScore{ 0 }; - - if (producerCurrentRtpStream) - { - producerScore = producerCurrentRtpStream->GetScore(); - } - else - { - producerScore = 0; - } - - return FBS::Consumer::CreateConsumerScoreDirect( - builder, this->rtpStream->GetScore(), producerScore, this->producerRtpStreamScores); - } - - void SimulcastConsumer::HandleRequest(Channel::ChannelRequest* request) - { - MS_TRACE(); - - switch (request->method) - { - case Channel::ChannelRequest::Method::CONSUMER_DUMP: - { - auto dumpOffset = FillBuffer(request->GetBufferBuilder()); - - request->Accept(FBS::Response::Body::Consumer_DumpResponse, dumpOffset); - - break; - } - - case Channel::ChannelRequest::Method::CONSUMER_REQUEST_KEY_FRAME: - { - if (IsActive()) - { - RequestKeyFrames(); - } - - request->Accept(); - - break; - } - - case Channel::ChannelRequest::Method::CONSUMER_SET_PREFERRED_LAYERS: - { - auto previousPreferredLayers = this->preferredLayers; - - const auto* body = request->data->body_as(); - const auto* preferredLayers = body->preferredLayers(); - - // Spatial layer. - this->preferredLayers.spatial = preferredLayers->spatialLayer(); - - if (this->preferredLayers.spatial > this->rtpStream->GetSpatialLayers() - 1) - { - this->preferredLayers.spatial = - static_cast(this->rtpStream->GetSpatialLayers() - 1); - } - - // preferredTemporaLayer is optional. - auto preferredTemporalLayer = preferredLayers->temporalLayer(); - - if (preferredTemporalLayer.has_value()) - { - this->preferredLayers.temporal = preferredTemporalLayer.value(); - - if (this->preferredLayers.temporal > this->rtpStream->GetTemporalLayers() - 1) - { - this->preferredLayers.temporal = - static_cast(this->rtpStream->GetTemporalLayers() - 1); - } - } - else - { - this->preferredLayers.temporal = - static_cast(this->rtpStream->GetTemporalLayers() - 1); - } - - MS_DEBUG_DEV( - "preferred layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", - this->preferredLayers.spatial, - this->preferredLayers.temporal, - this->id.c_str()); - - preferredTemporalLayer = this->preferredLayers.temporal; - auto preferredLayersOffset = FBS::Consumer::CreateConsumerLayers( - request->GetBufferBuilder(), this->preferredLayers.spatial, preferredTemporalLayer); - auto responseOffset = FBS::Consumer::CreateSetPreferredLayersResponse( - request->GetBufferBuilder(), preferredLayersOffset); - - request->Accept(FBS::Response::Body::Consumer_SetPreferredLayersResponse, responseOffset); - - if (IsActive() && this->preferredLayers != previousPreferredLayers) - { - MayChangeLayers(/*force*/ true); - } - - break; - } - - default: - { - // Pass it to the parent class. - RTC::Consumer::HandleRequest(request); - } - } - } - - void SimulcastConsumer::ProducerRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) - { - MS_TRACE(); - - auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); - - MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); - - const int16_t spatialLayer = it->second; - - this->producerRtpStreams[spatialLayer] = rtpStream; - } - - void SimulcastConsumer::ProducerNewRtpStream(RTC::RTP::RtpStreamRecv* rtpStream, uint32_t mappedSsrc) - { - MS_TRACE(); - - auto it = this->mapMappedSsrcSpatialLayer.find(mappedSsrc); - - MS_ASSERT(it != this->mapMappedSsrcSpatialLayer.end(), "unknown mappedSsrc"); - - const int16_t spatialLayer = it->second; - - this->producerRtpStreams[spatialLayer] = rtpStream; - - // Emit the score event. - EmitScore(); - - if (IsActive()) - { - MayChangeLayers(); - } - } - - void SimulcastConsumer::ProducerRtpStreamScore( - RTC::RTP::RtpStreamRecv* /*rtpStream*/, uint8_t score, uint8_t previousScore) - { - MS_TRACE(); - - // Emit the score event. - EmitScore(); - - if (RTC::Consumer::IsActive()) - { - // All Producer streams are dead. - if (!IsActive()) - { - UpdateTargetLayers(-1, -1); - } - // Just check target layers if the stream has died or reborned. - else if (!this->externallyManagedBitrate || (score == 0u || previousScore == 0u)) - { - MayChangeLayers(); - } - } - } - - void SimulcastConsumer::ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) - { - MS_TRACE(); - - // Just interested if this is the first Sender Report for a RTP stream. - if (!first) - { - return; - } - - MS_DEBUG_TAG(simulcast, "first SenderReport [ssrc:%" PRIu32 "]", rtpStream->GetSsrc()); - - // If our RTP timestamp reference stream does not yet have SR, do nothing - // since we know we won't be able to switch. - auto* producerTsReferenceRtpStream = GetProducerTsReferenceRtpStream(); - - if (!producerTsReferenceRtpStream || !producerTsReferenceRtpStream->GetSenderReportNtpMs()) - { - return; - } - - if (IsActive()) - { - MayChangeLayers(); - } - } - - uint8_t SimulcastConsumer::GetBitratePriority() const - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - - if (!IsActive()) - { - return 0u; - } - - return this->priority; - } - - uint32_t SimulcastConsumer::IncreaseLayer(uint32_t bitrate, bool considerLoss) - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - MS_ASSERT(IsActive(), "should be active"); - - // If already in the preferred layers, do nothing. - if (this->provisionalTargetLayers == this->preferredLayers) - { - return 0u; - } - - uint32_t virtualBitrate; - - if (considerLoss) - { - // Calculate virtual available bitrate based on given bitrate and our - // packet lost. - auto lossPercentage = this->rtpStream->GetLossPercentage(); - - if (lossPercentage < 2) - { - virtualBitrate = 1.08 * bitrate; - } - else if (lossPercentage > 10) - { - virtualBitrate = (1 - 0.5 * (lossPercentage / 100)) * bitrate; - } - else - { - virtualBitrate = bitrate; - } - } - else - { - virtualBitrate = bitrate; - } - - uint32_t requiredBitrate{ 0u }; - int16_t spatialLayer{ 0 }; - int16_t temporalLayer{ 0 }; - auto nowMs = DepLibUV::GetTimeMs(); - - for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx) - { - spatialLayer = static_cast(sIdx); - - // If this is higher than current spatial layer and we moved to to current - // spatial layer due to BWE limitations, check how much it has elapsed - // since then. - if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) - { - if (this->provisionalTargetLayers.spatial > -1 && spatialLayer > this->currentSpatialLayer) - { - MS_DEBUG_DEV( - "avoid upgrading to spatial layer %" PRIi16 " due to recent BWE downgrade", spatialLayer); - - goto done; - } - } - - // Ignore spatial layers lower than the one we already have. - if (spatialLayer < this->provisionalTargetLayers.spatial) - { - continue; - } - // If this is the higher than preferred spatial layer, abort. - else if (spatialLayer > this->preferredLayers.spatial) - { - MS_DEBUG_DEV( - "avoid upgrading to spatial layer %" PRIi16 - " since it's higher than preferred spatial layer %" PRIi16, - spatialLayer, - this->preferredLayers.spatial); - - goto done; - } - - // This can be null. - auto* producerRtpStream = this->producerRtpStreams.at(spatialLayer); - - // Producer stream does not exist. Ignore. - if (!producerRtpStream) - { - continue; - } - - // Ignore spatial layers (streams) with score 0. - if (producerRtpStream->GetScore() == 0) - { - continue; - } - - // If the stream has not been active time enough and we have an active one - // already, move to the next spatial layer. - if ( - spatialLayer != this->provisionalTargetLayers.spatial && - this->provisionalTargetLayers.spatial != -1 && - producerRtpStream->GetActiveMs() < StreamMinActiveMs) - { - const auto* provisionalProducerRtpStream = - this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); - - // The stream for the current provisional spatial layer has been active - // for enough time, move to the next spatial layer. - if (provisionalProducerRtpStream->GetActiveMs() >= StreamMinActiveMs) - { - continue; - } - } - - // We may not yet switch to this spatial layer. - if (!CanSwitchToSpatialLayer(spatialLayer)) - { - continue; - } - - temporalLayer = 0; - - // Check bitrate of every temporal layer. - for (; temporalLayer < producerRtpStream->GetTemporalLayers(); ++temporalLayer) - { - // Ignore temporal layers lower than the one we already have (taking - // into account the spatial layer too). - if ( - spatialLayer == this->provisionalTargetLayers.spatial && - temporalLayer <= this->provisionalTargetLayers.temporal) - { - continue; - } - - requiredBitrate = producerRtpStream->GetLayerBitrate(nowMs, 0, temporalLayer); - - // This is simulcast so we must substract the bitrate of the current - // temporal spatial layer if this is the temporal layer 0 of a higher - // spatial layer. - if ( - requiredBitrate && temporalLayer == 0 && this->provisionalTargetLayers.spatial > -1 && - spatialLayer > this->provisionalTargetLayers.spatial) - { - auto* provisionalProducerRtpStream = - this->producerRtpStreams.at(this->provisionalTargetLayers.spatial); - auto provisionalRequiredBitrate = provisionalProducerRtpStream->GetBitrate( - nowMs, 0, this->provisionalTargetLayers.temporal); - - if (requiredBitrate > provisionalRequiredBitrate) - { - requiredBitrate -= provisionalRequiredBitrate; - } - else - { - requiredBitrate = 1u; // Don't set 0 since it would be ignored. - } - } - - MS_DEBUG_DEV( - "testing layers %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 - ", required bitrate:%" PRIu32 "]", - spatialLayer, - temporalLayer, - virtualBitrate, - requiredBitrate); - - // If active layer, end iterations here. Otherwise move to next spatial - // layer. - if (requiredBitrate) - { - goto done; - } - else - { - break; - } - } - - // If this is the preferred spatial layer or higher, take it and exit. - if (spatialLayer >= this->preferredLayers.spatial) - { - break; - } - } - - done: - - // No higher active layers found. - if (!requiredBitrate) - { - return 0u; - } - - // No luck. - if (requiredBitrate > virtualBitrate) - { - return 0u; - } - - // Set provisional layers. - this->provisionalTargetLayers.spatial = spatialLayer; - this->provisionalTargetLayers.temporal = temporalLayer; - - MS_DEBUG_DEV( - "setting provisional layers to %" PRIi16 ":%" PRIi16 " [virtual bitrate:%" PRIu32 - ", required bitrate:%" PRIu32 "]", - this->provisionalTargetLayers.spatial, - this->provisionalTargetLayers.temporal, - virtualBitrate, - requiredBitrate); - - if (requiredBitrate <= bitrate) - { - return requiredBitrate; - } - else if (requiredBitrate <= virtualBitrate) - { - return bitrate; - } - else - { - return requiredBitrate; // NOTE: This cannot happen. - } - } - - void SimulcastConsumer::ApplyLayers() - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - MS_ASSERT(IsActive(), "should be active"); - - auto provisionalTargetLayers = this->provisionalTargetLayers; - - // Reset provisional target layers. - this->provisionalTargetLayers.Reset(); - - if (provisionalTargetLayers != this->targetLayers) - { - UpdateTargetLayers(provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); - - // If this looks like a spatial layer downgrade due to BWE limitations, set - // member. - if ( - this->rtpStream->GetActiveMs() > BweDowngradeMinActiveMs && - this->targetLayers.spatial < this->currentSpatialLayer && - this->currentSpatialLayer <= this->preferredLayers.spatial) - { - MS_DEBUG_DEV( - "possible target spatial layer downgrade (from %" PRIi16 " to %" PRIi16 - ") due to BWE limitation", - this->currentSpatialLayer, - this->targetLayers.spatial); - - this->lastBweDowngradeAtMs = DepLibUV::GetTimeMs(); - } - } - } - - uint32_t SimulcastConsumer::GetDesiredBitrate() const - { - MS_TRACE(); - - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - - if (!IsActive()) - { - return 0u; - } - - auto nowMs = DepLibUV::GetTimeMs(); - uint32_t desiredBitrate{ 0u }; - - // Let's iterate all streams of the Producer (from highest to lowest) and - // obtain their bitrate. Choose the highest one. - // NOTE: When the Producer enables a higher stream, initially the bitrate of - // it could be less than the bitrate of a lower stream. That's why we - // iterate all streams here anyway. - for (auto sIdx{ static_cast(this->producerRtpStreams.size() - 1) }; sIdx >= 0; --sIdx) - { - auto* producerRtpStream = this->producerRtpStreams.at(sIdx); - - if (!producerRtpStream) - { - continue; - } - - auto streamBitrate = producerRtpStream->GetBitrate(nowMs); - - desiredBitrate = std::max(streamBitrate, desiredBitrate); - } - - // If consumer.rtpParameters.encodings[0].maxBitrate was given and it's - // greater than computed one, then use it. - auto maxBitrate = this->rtpParameters.encodings[0].maxBitrate; - - desiredBitrate = std::max(maxBitrate, desiredBitrate); - - return desiredBitrate; - } - - // NOLINTNEXTLINE(misc-no-recursion) - void SimulcastConsumer::SendRtpPacket(RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) - { - MS_TRACE(); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.consumerId = this->id; -#endif - - auto spatialLayer = this->mapMappedSsrcSpatialLayer.at(packet->GetSsrc()); - - if (!IsActive()) - { - // Only drop the packet in the RTP sequence manager if it belongs to the - // current spatial layer. - if (spatialLayer == this->currentSpatialLayer) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::CONSUMER_INACTIVE); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - } - - return; - } - - if (this->targetLayers.temporal == -1) - { - // Only drop the packet in the RTP sequence manager if it belongs to the - // current spatial layer. - if (spatialLayer == this->currentSpatialLayer) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::INVALID_TARGET_LAYER); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - } - - return; - } - - auto payloadType = packet->GetPayloadType(); - - // NOTE: This may happen if this Consumer supports just some codecs of those - // in the corresponding Producer. - if (!this->supportedCodecPayloadTypes[payloadType]) - { - // Only drop the packet in the RTP sequence manager if it belongs to the - // current spatial layer. - if (spatialLayer == this->currentSpatialLayer) - { - MS_WARN_DEV("payload type not supported [payloadType:%" PRIu8 "]", payloadType); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::UNSUPPORTED_PAYLOAD_TYPE); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - } - - return; - } - - bool shouldSwitchCurrentSpatialLayer{ false }; - - // Check whether this is the packet we are waiting for in order to update - // the current spatial layer. - if ( - this->currentSpatialLayer != this->targetLayers.spatial && - spatialLayer == this->targetLayers.spatial) - { - // Ignore if not a key frame. - if (!packet->IsKeyFrame()) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); -#endif - - // NOTE: Don't drop the packet in the RTP sequence manager since this - // packet doesn't belong to the current spatial layer. - - // Store the packet for the scenario in which this packet is part of the - // key frame and it arrived before the first packet of the key frame. - StorePacketInTargetLayerRetransmissionBuffer(packet, sharedPacket); - - return; - } - - shouldSwitchCurrentSpatialLayer = true; - - // Need to resync the stream. - this->syncRequired = true; - this->spatialLayerToSync = spatialLayer; - } - // If the packet belongs to different spatial layer than the one being sent, - // drop it. - else if (spatialLayer != this->currentSpatialLayer) - { - // NOTE: Don't drop the packet in the RTP sequence manager since this - // packet doesn't belong to the current spatial layer. - - return; - } - - // If we need to sync and this is not a key frame, ignore the packet. - // NOTE: syncRequired is true if packet is a key frame of the target spatial - // layer or if transport just connected or consumer resumed. - if (this->syncRequired && !packet->IsKeyFrame()) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); -#endif - - // NOTE: No need to drop the packet in the RTP sequence manager since here - // we are blocking all packets but the key frame that would trigger sync - // below. - - // Store the packet for the scenario in which this packet is part of the - // key frame and it arrived before the first packet of the key frame. - StorePacketInTargetLayerRetransmissionBuffer(packet, sharedPacket); - - return; - } - - // Packets with only padding are not forwarded. - if (packet->GetPayloadLength() == 0) - { - // Only drop the packet in the RTP sequence manager if it belongs to the - // current spatial layer. - if (spatialLayer == this->currentSpatialLayer) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - } - - return; - } - - // Whether this is the first packet after re-sync. - const bool isSyncPacket = this->syncRequired; - - // Whether packets stored in the target layer retransmission buffer must be - // sent once this packet is sent. - bool sendPacketsInTargetLayerRetransmissionBuffer{ false }; - - // Sync sequence number and timestamp if required. - if (isSyncPacket && (this->spatialLayerToSync == -1 || spatialLayer == this->spatialLayerToSync)) - { - if (packet->IsKeyFrame()) - { - MS_DEBUG_TAG( - rtp, - "sync key frame received [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - - sendPacketsInTargetLayerRetransmissionBuffer = true; - } - - uint32_t tsOffset{ 0u }; - - // Sync our RTP stream's RTP timestamp. - if (spatialLayer == this->tsReferenceSpatialLayer) - { - tsOffset = 0u; - } - // If this is not the RTP stream we use as TS reference, do NTP based RTP - // TS synchronization. - else - { - auto* producerTsReferenceRtpStream = GetProducerTsReferenceRtpStream(); - auto* producerTargetRtpStream = GetProducerTargetRtpStream(); - - // NOTE: If we are here is because we have Sender Reports for both the - // TS reference stream and the target one. - MS_ASSERT( - producerTsReferenceRtpStream->GetSenderReportNtpMs(), - "no Sender Report for TS reference RTP stream"); - MS_ASSERT( - producerTargetRtpStream->GetSenderReportNtpMs(), "no Sender Report for current RTP stream"); - - // Calculate NTP and TS stuff. - auto ntpMs1 = producerTsReferenceRtpStream->GetSenderReportNtpMs(); - auto ts1 = producerTsReferenceRtpStream->GetSenderReportTs(); - auto ntpMs2 = producerTargetRtpStream->GetSenderReportNtpMs(); - auto ts2 = producerTargetRtpStream->GetSenderReportTs(); - int64_t diffMs; - - if (ntpMs2 >= ntpMs1) - { - diffMs = ntpMs2 - ntpMs1; - } - else - { - diffMs = -1 * (ntpMs1 - ntpMs2); - } - - const int64_t diffTs = diffMs * this->rtpStream->GetClockRate() / 1000; - const uint32_t newTs2 = ts2 - diffTs; - - // Apply offset. This is the difference that later must be removed from the - // sending RTP packet. - tsOffset = newTs2 - ts1; - } - - // When switching to a new stream it may happen that the timestamp of this - // key frame is lower than the highest timestamp sent to the remote endpoint. - // If so, apply an extra offset to "fix" it for the whole live of this - // selected Producer stream. - if (shouldSwitchCurrentSpatialLayer && (packet->GetTimestamp() - tsOffset <= this->rtpStream->GetMaxPacketTs())) - { - // Max delay in ms we allow for the stream when switching. - // https://en.wikipedia.org/wiki/Audio-to-video_synchronization#Recommendations - static const uint32_t MaxExtraOffsetMs{ 75u }; - - // Outgoing packet matches the highest timestamp seen in the previous - // stream. Apply an expected offset for a new frame in a 30fps stream. - static const uint8_t MsOffset{ 33u }; // (1 / 30 * 1000). - - const int64_t maxTsExtraOffset = MaxExtraOffsetMs * this->rtpStream->GetClockRate() / 1000; - uint32_t tsExtraOffset = this->rtpStream->GetMaxPacketTs() - packet->GetTimestamp() + - tsOffset + (MsOffset * this->rtpStream->GetClockRate() / 1000); - - // NOTE: Don't ask for a key frame if already done. - if (this->keyFrameForTsOffsetRequested) - { - // Give up and use the theoretical offset. - if (tsExtraOffset > maxTsExtraOffset) - { - MS_WARN_TAG( - simulcast, - "giving up on proper stream switching after got a requested keyframe for which still too high RTP timestamp extra offset is needed (%" PRIu32 - ")", - tsExtraOffset); - - tsExtraOffset = 1u; - } - } - else if (tsExtraOffset > maxTsExtraOffset) - { - MS_WARN_TAG( - simulcast, - "cannot switch stream due to too high RTP timestamp extra offset needed (%" PRIu32 - "), requesting keyframe", - tsExtraOffset); - - RequestKeyFrameForTargetSpatialLayer(); - - this->keyFrameForTsOffsetRequested = true; - - // Reset flags since we are discarding this key frame. - this->syncRequired = false; - this->spatialLayerToSync = -1; - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded( - RTC::RtcLogger::RtpPacket::DiscardReason::TOO_HIGH_TIMESTAMP_EXTRA_NEEDED); -#endif - - // NOTE: Don't drop the packet in the RTP sequence manager since this - // packet doesn't belong to the current spatial layer. - - return; - } - - if (tsExtraOffset > 0u) - { - MS_DEBUG_TAG( - simulcast, - "RTP timestamp extra offset generated for stream switching: %" PRIu32, - tsExtraOffset); - - // Increase the timestamp offset for the whole life of this Producer stream - // (until switched to a different one). - tsOffset -= tsExtraOffset; - } - } - - this->tsOffset = tsOffset; - - // Sync our RTP stream's sequence number. - // If previous frame has not been sent completely when we switch layer, - // we can tell libwebrtc that previous frame is incomplete by skipping - // one RTP sequence number. - // 'packet->GetSequenceNumber() -2' may increase SeqManager::base and - // increase the output sequence number. - // https://github.com/versatica/mediasoup/issues/408 - this->rtpSeqManager.Sync(packet->GetSequenceNumber() - (this->lastSentPacketHasMarker ? 1 : 2)); - - this->encodingContext->SyncRequired(); - - this->syncRequired = false; - this->spatialLayerToSync = -1; - this->keyFrameForTsOffsetRequested = false; - } - - if (!shouldSwitchCurrentSpatialLayer && this->checkingForOldPacketsInSpatialLayer) - { - // If this is a packet previous to the spatial layer switch, ignore the - // packet. - // NOTE: We drop it in RTP sequence manager because this packet belongs - // to current spatial layer. - if (SeqManager::IsSeqLowerThan( - packet->GetSequenceNumber(), this->snReferenceSpatialLayer)) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded( - RTC::RtcLogger::RtpPacket::DiscardReason::PACKET_PREVIOUS_TO_SPATIAL_LAYER_SWITCH); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - - return; - } - else if (SeqManager::IsSeqHigherThan( - packet->GetSequenceNumber(), this->snReferenceSpatialLayer + MaxSequenceNumberGap)) - { - this->checkingForOldPacketsInSpatialLayer = false; - } - } - - bool marker{ false }; - - if (shouldSwitchCurrentSpatialLayer) - { - // Update current spatial layer. - this->currentSpatialLayer = this->targetLayers.spatial; - - this->snReferenceSpatialLayer = packet->GetSequenceNumber(); - this->checkingForOldPacketsInSpatialLayer = true; - - // Update target and current temporal layer. - this->encodingContext->SetTargetTemporalLayer(this->targetLayers.temporal); - this->encodingContext->SetCurrentTemporalLayer(packet->GetTemporalLayer()); - - // Reset the score of our RtpStream to 10. - this->rtpStream->ResetScore(10u, /*notify*/ false); - - // Emit the layersChange event. - EmitLayersChange(); - - // Emit the score event. - EmitScore(); - - // Rewrite payload if needed. - packet->ProcessPayload(this->encodingContext.get(), marker); - } - else - { - auto previousTemporalLayer = this->encodingContext->GetCurrentTemporalLayer(); - - // Rewrite payload if needed. Drop packet if necessary. - // NOTE: We drop it in RTP sequence manager because this packet belongs - // to current spatial layer. - if (!packet->ProcessPayload(this->encodingContext.get(), marker)) - { -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::DROPPED_BY_CODEC); -#endif - - this->rtpSeqManager.Drop(packet->GetSequenceNumber()); - - return; - } - - if (previousTemporalLayer != this->encodingContext->GetCurrentTemporalLayer()) - { - EmitLayersChange(); - } - } - - // Update RTP seq number and timestamp based on NTP offset. - uint16_t seq; - const uint32_t timestamp = packet->GetTimestamp() - this->tsOffset; - - this->rtpSeqManager.Input(packet->GetSequenceNumber(), seq); - - // Save original packet fields. - auto origSsrc = packet->GetSsrc(); - auto origSeq = packet->GetSequenceNumber(); - auto origTimestamp = packet->GetTimestamp(); - - // Rewrite packet. - packet->SetSsrc(this->rtpParameters.encodings[0].ssrc); - packet->SetSequenceNumber(seq); - packet->SetTimestamp(timestamp); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.sendRtpTimestamp = timestamp; - packet->logger.sendSeqNumber = seq; -#endif - - if (isSyncPacket) - { - MS_DEBUG_TAG( - rtp, - "sending sync packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 - "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp(), - origSsrc, - origSeq, - origTimestamp); - } - - const RTC::RTP::RtpStreamSend::ReceivePacketResult result = - this->rtpStream->ReceivePacket(packet, sharedPacket); - - if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) - { - if (this->rtpSeqManager.GetMaxOutput() == packet->GetSequenceNumber()) - { - this->lastSentPacketHasMarker = packet->HasMarker(); - } - - // Send the packet. - this->listener->OnConsumerSendRtpPacket(this, packet); - - // May emit 'trace' event. - EmitTraceEventRtpAndKeyFrameTypes(packet); - } - else - { - MS_WARN_TAG( - rtp, - "failed to send packet [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 - "] from original [ssrc:%" PRIu32 ", seq:%" PRIu16 ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp(), - origSsrc, - origSeq, - origTimestamp); - -#ifdef MS_RTC_LOGGER_RTP - packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::SEND_RTP_STREAM_DISCARDED); -#endif - } - - // Restore packet fields. - packet->SetSsrc(origSsrc); - packet->SetSequenceNumber(origSeq); - packet->SetTimestamp(origTimestamp); - - // Restore the original payload if needed. - packet->RestorePayload(); - - // If sharedPacket doesn't have a packet inside and it has been stored we - // need to clone the packet into it. - if (!sharedPacket.HasPacket() && result == RTC::RTP::RtpStreamSend::ReceivePacketResult::ACCEPTED_AND_STORED) - { - sharedPacket.Assign(packet); - } - - // If sent packet was the first packet of a key frame, let's send buffered - // packets belonging to the same key frame that arrived earlier due to - // packet misorder. - if (sendPacketsInTargetLayerRetransmissionBuffer) - { - // NOTE: Only send buffered packets if the first packet containing the key - // frame was sent. - if (result != RTC::RTP::RtpStreamSend::ReceivePacketResult::DISCARDED) - { - for (auto& kv : this->targetLayerRetransmissionBuffer) - { - auto& bufferedSharedPacket = kv.second; - auto* bufferedPacket = bufferedSharedPacket.GetPacket(); - - if (bufferedPacket->GetSequenceNumber() > origSeq) - { - MS_DEBUG_DEV( - "sending packet buffered in the target layer retransmission buffer [ssrc:%" PRIu32 - ", seq:%" PRIu16 ", ts:%" PRIu32 - "] after sending first packet of the key frame [ssrc:%" PRIu32 ", seq:%" PRIu16 - ", ts:%" PRIu32 "]", - bufferedPacket->GetSsrc(), - bufferedPacket->GetSequenceNumber(), - bufferedPacket->GetTimestamp(), - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - - SendRtpPacket(bufferedPacket, bufferedSharedPacket); - - // Be sure that the target layer retransmission buffer has not been - // emptied as a result of sending this packet. If so, exit the loop. - if (this->targetLayerRetransmissionBuffer.empty()) - { - MS_DEBUG_DEV( - "target layer retransmission buffer emptied while iterating it, exiting the loop"); - - break; - } - } - } - } - - this->targetLayerRetransmissionBuffer.clear(); - } - } - - bool SimulcastConsumer::GetRtcp(RTC::RTCP::CompoundPacket* packet, uint64_t nowMs) - { - MS_TRACE(); - - if (static_cast((nowMs - this->lastRtcpSentTime) * 1.15) < this->maxRtcpInterval) - { - return true; - } - - auto* senderReport = this->rtpStream->GetRtcpSenderReport(nowMs); - - if (!senderReport) - { - return true; - } - - // Build SDES chunk for this sender. - auto* sdesChunk = this->rtpStream->GetRtcpSdesChunk(); - - auto* delaySinceLastRrSsrcInfo = this->rtpStream->GetRtcpXrDelaySinceLastRrSsrcInfo(nowMs); - - // RTCP Compound packet buffer cannot hold the data. - if (!packet->Add(senderReport, sdesChunk, delaySinceLastRrSsrcInfo)) - { - return false; - } - - this->lastRtcpSentTime = nowMs; - - return true; - } - - void SimulcastConsumer::NeedWorstRemoteFractionLost( - uint32_t /*mappedSsrc*/, uint8_t& worstRemoteFractionLost) - { - MS_TRACE(); - - if (!IsActive()) - { - return; - } - - auto fractionLost = this->rtpStream->GetFractionLost(); - - // If our fraction lost is worse than the given one, update it. - worstRemoteFractionLost = std::max(fractionLost, worstRemoteFractionLost); - } - - void SimulcastConsumer::ReceiveNack(RTC::RTCP::FeedbackRtpNackPacket* nackPacket) - { - MS_TRACE(); - - if (!IsActive()) - { - return; - } - - // May emit 'trace' event. - EmitTraceEventNackType(); - - this->rtpStream->ReceiveNack(nackPacket); - } - - void SimulcastConsumer::ReceiveKeyFrameRequest( - RTC::RTCP::FeedbackPs::MessageType messageType, uint32_t ssrc) - { - MS_TRACE(); - - switch (messageType) - { - case RTC::RTCP::FeedbackPs::MessageType::PLI: - { - EmitTraceEventPliType(ssrc); - - break; - } - - case RTC::RTCP::FeedbackPs::MessageType::FIR: - { - EmitTraceEventFirType(ssrc); - - break; - } - - default:; - } - - this->rtpStream->ReceiveKeyFrameRequest(messageType); - - if (IsActive()) - { - RequestKeyFrameForCurrentSpatialLayer(); - } - } - - void SimulcastConsumer::ReceiveRtcpReceiverReport(RTC::RTCP::ReceiverReport* report) - { - MS_TRACE(); - - this->rtpStream->ReceiveRtcpReceiverReport(report); - } - - void SimulcastConsumer::ReceiveRtcpXrReceiverReferenceTime(RTC::RTCP::ReceiverReferenceTime* report) - { - MS_TRACE(); - - this->rtpStream->ReceiveRtcpXrReceiverReferenceTime(report); - } - - uint32_t SimulcastConsumer::GetTransmissionRate(uint64_t nowMs) - { - MS_TRACE(); - - if (!IsActive()) - { - return 0u; - } - - return this->rtpStream->GetBitrate(nowMs); - } - - float SimulcastConsumer::GetRtt() const - { - MS_TRACE(); - - return this->rtpStream->GetRtt(); - } - - void SimulcastConsumer::UserOnTransportConnected() - { - MS_TRACE(); - - this->syncRequired = true; - this->spatialLayerToSync = -1; - this->keyFrameForTsOffsetRequested = false; - - if (IsActive()) - { - MayChangeLayers(); - } - } - - void SimulcastConsumer::UserOnTransportDisconnected() - { - MS_TRACE(); - - this->lastBweDowngradeAtMs = 0u; - - this->rtpStream->Pause(); - this->targetLayerRetransmissionBuffer.clear(); - - UpdateTargetLayers(-1, -1); - } - - void SimulcastConsumer::UserOnPaused() - { - MS_TRACE(); - - this->lastBweDowngradeAtMs = 0u; - - this->rtpStream->Pause(); - this->targetLayerRetransmissionBuffer.clear(); - - UpdateTargetLayers(-1, -1); - - if (this->externallyManagedBitrate) - { - this->listener->OnConsumerNeedZeroBitrate(this); - } - } - - void SimulcastConsumer::UserOnResumed() - { - MS_TRACE(); - - this->syncRequired = true; - this->spatialLayerToSync = -1; - this->keyFrameForTsOffsetRequested = false; - this->checkingForOldPacketsInSpatialLayer = false; - - if (IsActive()) - { - MayChangeLayers(); - } - } - - void SimulcastConsumer::CreateRtpStream() - { - MS_TRACE(); - - auto& encoding = this->rtpParameters.encodings[0]; - const auto* mediaCodec = this->rtpParameters.GetCodecForEncoding(encoding); - - MS_DEBUG_TAG( - rtp, "[ssrc:%" PRIu32 ", payloadType:%" PRIu8 "]", encoding.ssrc, mediaCodec->payloadType); - - // Set stream params. - RTC::RTP::RtpStream::Params params; - - params.ssrc = encoding.ssrc; - params.payloadType = mediaCodec->payloadType; - params.mimeType = mediaCodec->mimeType; - params.clockRate = mediaCodec->clockRate; - params.cname = this->rtpParameters.rtcp.cname; - params.spatialLayers = encoding.spatialLayers; - params.temporalLayers = encoding.temporalLayers; - - // Check in band FEC in codec parameters. - if (mediaCodec->parameters.HasInteger("useinbandfec") && mediaCodec->parameters.GetInteger("useinbandfec") == 1) - { - MS_DEBUG_TAG(rtp, "in band FEC enabled"); - - params.useInBandFec = true; - } - - // Check DTX in codec parameters. - if (mediaCodec->parameters.HasInteger("usedtx") && mediaCodec->parameters.GetInteger("usedtx") == 1) - { - MS_DEBUG_TAG(rtp, "DTX enabled"); - - params.useDtx = true; - } - - // Check DTX in the encoding. - if (encoding.dtx) - { - MS_DEBUG_TAG(rtp, "DTX enabled"); - - params.useDtx = true; - } - - for (const auto& fb : mediaCodec->rtcpFeedback) - { - if (!params.useNack && fb.type == "nack" && fb.parameter.empty()) - { - MS_DEBUG_2TAGS(rtp, rtcp, "NACK supported"); - - params.useNack = true; - } - else if (!params.usePli && fb.type == "nack" && fb.parameter == "pli") - { - MS_DEBUG_2TAGS(rtp, rtcp, "PLI supported"); - - params.usePli = true; - } - else if (!params.useFir && fb.type == "ccm" && fb.parameter == "fir") - { - MS_DEBUG_2TAGS(rtp, rtcp, "FIR supported"); - - params.useFir = true; - } - } - - this->rtpStream = new RTC::RTP::RtpStreamSend(this, params, this->rtpParameters.mid); - this->rtpStreams.push_back(this->rtpStream); - - // If the Consumer is paused, tell the RtpStreamSend. - if (IsPaused() || IsProducerPaused()) - { - this->rtpStream->Pause(); - } - - const auto* rtxCodec = this->rtpParameters.GetRtxCodecForEncoding(encoding); - - if (rtxCodec && encoding.hasRtx) - { - this->rtpStream->SetRtx(rtxCodec->payloadType, encoding.rtx.ssrc); - } - } - - void SimulcastConsumer::RequestKeyFrames() - { - MS_TRACE(); - - if (this->kind != RTC::Media::Kind::VIDEO) - { - return; - } - - auto* producerTargetRtpStream = GetProducerTargetRtpStream(); - auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); - - if (producerTargetRtpStream) - { - auto mappedSsrc = this->consumableRtpEncodings[this->targetLayers.spatial].ssrc; - - this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); - } - - if (producerCurrentRtpStream && producerCurrentRtpStream != producerTargetRtpStream) - { - auto mappedSsrc = this->consumableRtpEncodings[this->currentSpatialLayer].ssrc; - - this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); - } - } - - void SimulcastConsumer::RequestKeyFrameForTargetSpatialLayer() - { - MS_TRACE(); - - if (this->kind != RTC::Media::Kind::VIDEO) - { - return; - } - - auto* producerTargetRtpStream = GetProducerTargetRtpStream(); - - if (!producerTargetRtpStream) - { - return; - } - - auto mappedSsrc = this->consumableRtpEncodings[this->targetLayers.spatial].ssrc; - - this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); - } - - void SimulcastConsumer::RequestKeyFrameForCurrentSpatialLayer() - { - MS_TRACE(); - - if (this->kind != RTC::Media::Kind::VIDEO) - { - return; - } - - auto* producerCurrentRtpStream = GetProducerCurrentRtpStream(); - - if (!producerCurrentRtpStream) - { - return; - } - - auto mappedSsrc = this->consumableRtpEncodings[this->currentSpatialLayer].ssrc; - - this->listener->OnConsumerKeyFrameRequested(this, mappedSsrc); - } - - void SimulcastConsumer::MayChangeLayers(bool force) - { - MS_TRACE(); - - RTC::ConsumerTypes::VideoLayers newTargetLayers; - - if (RecalculateTargetLayers(newTargetLayers)) - { - // If bitrate externally managed, don't bother the transport unless - // the newTargetSpatialLayer has changed (or force is true). - // This is because, if bitrate is externally managed, the target temporal - // layer is managed by the available given bitrate so the transport - // will let us change it when it considers. - if (this->externallyManagedBitrate) - { - if (newTargetLayers.spatial != this->targetLayers.spatial || force) - { - this->listener->OnConsumerNeedBitrateChange(this); - } - } - else - { - UpdateTargetLayers(newTargetLayers.spatial, newTargetLayers.temporal); - } - } - } - - bool SimulcastConsumer::RecalculateTargetLayers(RTC::ConsumerTypes::VideoLayers& newTargetLayers) const - { - MS_TRACE(); - - // Start with no layers. - newTargetLayers.Reset(); - - auto nowMs = DepLibUV::GetTimeMs(); - - for (size_t sIdx{ 0u }; sIdx < this->producerRtpStreams.size(); ++sIdx) - { - auto spatialLayer = static_cast(sIdx); - auto* producerRtpStream = this->producerRtpStreams.at(sIdx); - auto producerScore = producerRtpStream ? producerRtpStream->GetScore() : 0u; - - // If this is higher than current spatial layer and we moved to to current spatial - // layer due to BWE limitations, check how much it has elapsed since then. - if (nowMs - this->lastBweDowngradeAtMs < BweDowngradeConservativeMs) - { - if (newTargetLayers.spatial > -1 && spatialLayer > this->currentSpatialLayer) - { - continue; - } - } - - // Ignore spatial layers for non existing Producer streams or for those - // with score 0. - if (producerScore == 0u) - { - continue; - } - - // If the stream has not been active time enough and we have an active one - // already, move to the next spatial layer. - // NOTE: Require bitrate externally managed for this. - if (this->externallyManagedBitrate && newTargetLayers.spatial != -1 && producerRtpStream->GetActiveMs() < StreamMinActiveMs) - { - continue; - } - - // We may not yet switch to this spatial layer. - if (!CanSwitchToSpatialLayer(spatialLayer)) - { - continue; - } - - newTargetLayers.spatial = spatialLayer; - - // If this is the preferred or higher spatial layer take it and exit. - if (spatialLayer >= this->preferredLayers.spatial) - { - break; - } - } - - if (newTargetLayers.spatial != -1) - { - if (newTargetLayers.spatial == this->preferredLayers.spatial) - { - newTargetLayers.temporal = this->preferredLayers.temporal; - } - else if (newTargetLayers.spatial < this->preferredLayers.spatial) - { - newTargetLayers.temporal = static_cast(this->rtpStream->GetTemporalLayers() - 1); - } - else - { - newTargetLayers.temporal = 0; - } - } - - // Return true if any target layer changed. - return (newTargetLayers != this->targetLayers); - } - - void SimulcastConsumer::UpdateTargetLayers(int16_t newTargetSpatialLayer, int16_t newTargetTemporalLayer) - { - MS_TRACE(); - - // If we don't have yet a RTP timestamp reference, set it now. - if ( - newTargetSpatialLayer != -1 && (this->tsReferenceSpatialLayer == -1 || - !GetProducerTsReferenceRtpStream()->GetSenderReportNtpMs())) - { - MS_DEBUG_TAG( - simulcast, "using spatial layer %" PRIi16 " as RTP timestamp reference", newTargetSpatialLayer); - - this->tsReferenceSpatialLayer = newTargetSpatialLayer; - } - - // If the new target spatial layer doesn't match the current one, clear the - // target layer retransmission buffer. - if (newTargetSpatialLayer != this->targetLayers.spatial) - { - this->targetLayerRetransmissionBuffer.clear(); - } - - if (newTargetSpatialLayer == -1) - { - // Unset current and target layers. - this->targetLayers.spatial = -1; - this->targetLayers.temporal = -1; - this->currentSpatialLayer = -1; - - this->encodingContext->SetTargetTemporalLayer(-1); - this->encodingContext->SetCurrentTemporalLayer(-1); - - MS_DEBUG_TAG( - simulcast, "target layers changed [spatial:-1, temporal:-1, consumerId:%s]", this->id.c_str()); - - EmitLayersChange(); - - return; - } - - this->targetLayers.spatial = newTargetSpatialLayer; - this->targetLayers.temporal = newTargetTemporalLayer; - - // If the new target spatial layer matches the current one, apply the new - // target temporal layer now. - if (this->targetLayers.spatial == this->currentSpatialLayer) - { - this->encodingContext->SetTargetTemporalLayer(this->targetLayers.temporal); - } - - MS_DEBUG_TAG( - simulcast, - "target layers changed [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", - this->targetLayers.spatial, - this->targetLayers.temporal, - this->id.c_str()); - - // If the target spatial layer is different than the current one, request - // a key frame. - if (this->targetLayers.spatial != this->currentSpatialLayer) - { - RequestKeyFrameForTargetSpatialLayer(); - } - } - - bool SimulcastConsumer::CanSwitchToSpatialLayer(int16_t spatialLayer) const - { - MS_TRACE(); - - // This method assumes that the caller has verified that there is a valid - // Producer RtpStream for the given spatial layer. - MS_ASSERT( - this->producerRtpStreams.at(spatialLayer), - "no Producer RtpStream for the given spatialLayer:%" PRIi16, - spatialLayer); - - // We can switch to the given spatial layer if: - // - we don't have any TS reference spatial layer yet, or - // - the given spatial layer matches the TS reference spatial layer, or - // - both , the RTP streams of our TS reference spatial layer and the given - // spatial layer, have Sender Report. - return ( - this->tsReferenceSpatialLayer == -1 || spatialLayer == this->tsReferenceSpatialLayer || - this->producerRtpStreams.at(spatialLayer)->GetSenderReportNtpMs()); - } - - void SimulcastConsumer::StorePacketInTargetLayerRetransmissionBuffer( - RTC::RTP::Packet* packet, RTC::RTP::SharedPacket& sharedPacket) - { - MS_TRACE(); - - MS_DEBUG_DEV( - "storing packet in target layer retransmission buffer [ssrc:%" PRIu32 ", seq:%" PRIu16 - ", ts:%" PRIu32 "]", - packet->GetSsrc(), - packet->GetSequenceNumber(), - packet->GetTimestamp()); - - // Store original packet into the buffer. Only clone once and only if - // necessary. - if (!sharedPacket.HasPacket()) - { - sharedPacket.Assign(packet); - } - // Assert that, if sharedPacket was already filled, both packet and - // sharedPacket are the very same RTP packet. - else - { - sharedPacket.AssertSamePacket(packet); - } - - this->targetLayerRetransmissionBuffer[packet->GetSequenceNumber()] = sharedPacket; - - if (this->targetLayerRetransmissionBuffer.size() > TargetLayerRetransmissionBufferSize) - { - this->targetLayerRetransmissionBuffer.erase(this->targetLayerRetransmissionBuffer.begin()); - } - } - - void SimulcastConsumer::EmitScore() const - { - MS_TRACE(); - - auto scoreOffset = FillBufferScore(this->shared->channelNotifier->GetBufferBuilder()); - - auto notificationOffset = FBS::Consumer::CreateScoreNotification( - this->shared->channelNotifier->GetBufferBuilder(), scoreOffset); - - this->shared->channelNotifier->Emit( - this->id, - FBS::Notification::Event::CONSUMER_SCORE, - FBS::Notification::Body::Consumer_ScoreNotification, - notificationOffset); - } - - void SimulcastConsumer::EmitLayersChange() const - { - MS_TRACE(); - - MS_DEBUG_DEV( - "current layers changed to [spatial:%" PRIi16 ", temporal:%" PRIi16 ", consumerId:%s]", - this->currentSpatialLayer, - this->encodingContext->GetCurrentTemporalLayer(), - this->id.c_str()); - - flatbuffers::Offset layersOffset; - - if (this->currentSpatialLayer >= 0) - { - layersOffset = FBS::Consumer::CreateConsumerLayers( - this->shared->channelNotifier->GetBufferBuilder(), - this->currentSpatialLayer, - this->encodingContext->GetCurrentTemporalLayer()); - } - - auto notificationOffset = FBS::Consumer::CreateLayersChangeNotification( - this->shared->channelNotifier->GetBufferBuilder(), layersOffset); - - this->shared->channelNotifier->Emit( - this->id, - FBS::Notification::Event::CONSUMER_LAYERS_CHANGE, - FBS::Notification::Body::Consumer_LayersChangeNotification, - notificationOffset); - } - - RTC::RTP::RtpStreamRecv* SimulcastConsumer::GetProducerCurrentRtpStream() const - { - MS_TRACE(); - - if (this->currentSpatialLayer == -1) - { - return nullptr; - } - - // This may return nullptr. - return this->producerRtpStreams.at(this->currentSpatialLayer); - } - - RTC::RTP::RtpStreamRecv* SimulcastConsumer::GetProducerTargetRtpStream() const - { - MS_TRACE(); - - if (this->targetLayers.spatial == -1) - { - return nullptr; - } - - // This may return nullptr. - return this->producerRtpStreams.at(this->targetLayers.spatial); - } - - RTC::RTP::RtpStreamRecv* SimulcastConsumer::GetProducerTsReferenceRtpStream() const - { - MS_TRACE(); - - if (this->tsReferenceSpatialLayer == -1) - { - return nullptr; - } - - // This may return nullptr. - return this->producerRtpStreams.at(this->tsReferenceSpatialLayer); - } - - void SimulcastConsumer::OnRtpStreamScore( - RTC::RTP::RtpStream* /*rtpStream*/, uint8_t /*score*/, uint8_t /*previousScore*/) - { - MS_TRACE(); - - // Emit the score event. - EmitScore(); - - if (IsActive()) - { - // Just check target layers if our bitrate is not externally managed. - // NOTE: For now this is a bit useless since, when locally managed, we do - // not check the Consumer score at all. - if (!this->externallyManagedBitrate) - { - MayChangeLayers(); - } - } - } - - void SimulcastConsumer::OnRtpStreamRetransmitRtpPacket( - RTC::RTP::RtpStreamSend* /*rtpStream*/, RTC::RTP::Packet* packet) - { - MS_TRACE(); - - this->listener->OnConsumerRetransmitRtpPacket(this, packet); - - // May emit 'trace' event. - EmitTraceEventRtpAndKeyFrameTypes(packet, this->rtpStream->HasRtx()); - } -} // namespace RTC diff --git a/worker/test/src/RTC/TestMultiStreamConsumer.cpp b/worker/test/src/RTC/TestMultiStreamConsumer.cpp index 3fd4cf3012..f01250be4a 100644 --- a/worker/test/src/RTC/TestMultiStreamConsumer.cpp +++ b/worker/test/src/RTC/TestMultiStreamConsumer.cpp @@ -169,9 +169,15 @@ namespace rtpStream(createRtpStreamRecv()) { // Set producer scores and producer stream. - const std::vector scores{ 10 }; + // NOTE: This must be static because the Consumer stores the given vector + // pointer which is supposed to exist in the associated Producer (but here + // there is no associated Producer). + static const std::vector scores{ 10 }; consumer->ProducerRtpStreamScores(&scores); + // NOTE: mappedSsrc here MUST be 1234567890 (otherwise Consumer will crash). + // This is guaranteed by Producer class, however here we must do it manually. + consumer->ProducerNewRtpStream(rtpStream.get(), 1234567890); } std::unique_ptr listener; From df0569bfeb728a6679eb08bd6383dfed5a7130be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Tue, 10 Feb 2026 19:20:56 +0100 Subject: [PATCH 05/10] tons of debugs --- worker/src/RTC/MultiStreamConsumer.cpp | 103 ++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp index f1d36d99e9..2f5a698b7c 100644 --- a/worker/src/RTC/MultiStreamConsumer.cpp +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -1,5 +1,6 @@ #define MS_CLASS "RTC::MultiStreamConsumer" -// #define MS_LOG_DEV_LEVEL 3 +// TODO +#define MS_LOG_DEV_LEVEL 3 #include "RTC/MultiStreamConsumer.hpp" #include "DepLibUV.hpp" @@ -366,6 +367,9 @@ namespace RTC // All Producer streams are dead. if (!IsActive()) { + // TODO: REMOVE + MS_DUMP("------ calling UpdateTargetLayers(-1, -1)"); + UpdateTargetLayers(-1, -1); } // Just check target layers if the stream has died or reborned. @@ -662,6 +666,9 @@ namespace RTC if (provisionalTargetLayers != this->targetLayers) { + // TODO: REMOVE + MS_DUMP("------ calling UpdateTargetLayers(%d, %d)", provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); + UpdateTargetLayers(provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); // If this looks like a spatial layer downgrade due to BWE limitations, set @@ -735,6 +742,12 @@ namespace RTC { MS_TRACE(); + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 1 audio packet | ssrc:%u", packet->GetSsrc()); + } + #ifdef MS_RTC_LOGGER_RTP packet->logger.consumerId = this->id; #endif @@ -743,6 +756,9 @@ namespace RTC if (!IsActive()) { + // TODO: REMOVE + MS_DUMP("----------- NOT ACTIVE !!!!"); + // Only drop the packet in the RTP sequence manager if it belongs to the // current spatial layer. if (spatialLayer == this->currentSpatialLayer) @@ -757,12 +773,20 @@ namespace RTC return; } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 2 audio packet | ssrc:%u", packet->GetSsrc()); + } + if (this->targetLayers.temporal == -1) { // Only drop the packet in the RTP sequence manager if it belongs to the // current spatial layer. if (spatialLayer == this->currentSpatialLayer) { + MS_DUMP("----- INVALID_TARGET_LAYER audio packet | spatialLayer (%u) == this->currentSpatialLayer (%u)", spatialLayer, this->currentSpatialLayer); + #ifdef MS_RTC_LOGGER_RTP packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::INVALID_TARGET_LAYER); #endif @@ -770,9 +794,21 @@ namespace RTC this->rtpSeqManager.Drop(packet->GetSequenceNumber()); } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- DROPPED audio packet | this->targetLayers.temporal == -1"); + } + return; } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 3 audio packet | ssrc:%u", packet->GetSsrc()); + } + auto payloadType = packet->GetPayloadType(); // NOTE: This may happen if this Consumer supports just some codecs of those @@ -795,6 +831,12 @@ namespace RTC return; } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 4 audio packet | ssrc:%u", packet->GetSsrc()); + } + bool shouldSwitchCurrentSpatialLayer{ false }; // Check whether this is the packet we are waiting for in order to update @@ -836,6 +878,12 @@ namespace RTC return; } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 5 audio packet | ssrc:%u", packet->GetSsrc()); + } + // If we need to sync and this is not a key frame, ignore the packet. // NOTE: syncRequired is true if packet is a key frame of the target spatial // layer or if transport just connected or consumer resumed. @@ -856,6 +904,12 @@ namespace RTC return; } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 6 audio packet | ssrc:%u", packet->GetSsrc()); + } + // Packets with only padding are not forwarded. if (packet->GetPayloadLength() == 0) { @@ -873,6 +927,12 @@ namespace RTC return; } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 7 audio packet | ssrc:%u", packet->GetSsrc()); + } + // Whether this is the first packet after re-sync. const bool isSyncPacket = this->syncRequired; @@ -1035,6 +1095,12 @@ namespace RTC this->keyFrameForTsOffsetRequested = false; } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 8 audio packet | ssrc:%u", packet->GetSsrc()); + } + if (!shouldSwitchCurrentSpatialLayer && this->checkingForOldPacketsInSpatialLayer) { // If this is a packet previous to the spatial layer switch, ignore the @@ -1060,10 +1126,19 @@ namespace RTC } } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 9 audio packet | ssrc:%u", packet->GetSsrc()); + } + bool marker{ false }; if (shouldSwitchCurrentSpatialLayer) { + // TODO: REMOVE + MS_DUMP("------ setting this->currentSpatialLayer = %d [previous value:%d]", this->targetLayers.spatial, this->currentSpatialLayer); + // Update current spatial layer. this->currentSpatialLayer = this->targetLayers.spatial; @@ -1116,6 +1191,12 @@ namespace RTC } } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- 10 audio packet | ssrc:%u", packet->GetSsrc()); + } + // Update RTP seq number and timestamp based on NTP offset. uint16_t seq; const uint32_t timestamp = packet->GetTimestamp() - this->tsOffset; @@ -1161,6 +1242,12 @@ namespace RTC this->lastSentPacketHasMarker = packet->HasMarker(); } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- SENT! audio packet | ssrc:%u", packet->GetSsrc()); + } + // Send the packet. this->listener->OnConsumerSendRtpPacket(this, packet); @@ -1398,6 +1485,9 @@ namespace RTC this->rtpStream->Pause(); this->targetLayerRetransmissionBuffer.clear(); + // TODO: REMOVE + MS_DUMP("------ calling UpdateTargetLayers(-1, -1) [current this->currentSpatialLayer:%d]", this->currentSpatialLayer); + UpdateTargetLayers(-1, -1); } @@ -1410,6 +1500,9 @@ namespace RTC this->rtpStream->Pause(); this->targetLayerRetransmissionBuffer.clear(); + // TODO: REMOVE + MS_DUMP("------ calling UpdateTargetLayers(-1, -1) [current this->currentSpatialLayer:%d]", this->currentSpatialLayer); + UpdateTargetLayers(-1, -1); if (this->externallyManagedBitrate) @@ -1608,6 +1701,9 @@ namespace RTC } else { + // TODO: REMOVE + MS_DUMP("------ calling UpdateTargetLayers(%d, %d)", newTargetLayers.spatial, newTargetLayers.temporal); + UpdateTargetLayers(newTargetLayers.spatial, newTargetLayers.temporal); } } @@ -1713,6 +1809,9 @@ namespace RTC if (newTargetSpatialLayer == -1) { + // TODO: REMOVE + MS_DUMP("------ setting this->currentSpatialLayer = -1"); + // Unset current and target layers. this->targetLayers.spatial = -1; this->targetLayers.temporal = -1; @@ -1863,6 +1962,8 @@ namespace RTC if (this->currentSpatialLayer == -1) { + // TODO: Remove + MS_DUMP("------ this->currentSpatialLayer == -1 => nullptr"); return nullptr; } From 21432c685a9c7beba7a87f2d6c0332560415bcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Tue, 10 Feb 2026 19:32:22 +0100 Subject: [PATCH 06/10] foo --- worker/src/RTC/MultiStreamConsumer.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp index 2f5a698b7c..36adc832af 100644 --- a/worker/src/RTC/MultiStreamConsumer.cpp +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -1810,7 +1810,7 @@ namespace RTC if (newTargetSpatialLayer == -1) { // TODO: REMOVE - MS_DUMP("------ setting this->currentSpatialLayer = -1"); + MS_DUMP("------ setting this->targetLayers.spatial = -1, this->targetLayers.temporal = -1, this->currentSpatialLayer = -1"); // Unset current and target layers. this->targetLayers.spatial = -1; @@ -1831,6 +1831,9 @@ namespace RTC return; } + // TODO + MS_DUMP("---- setting this->targetLayers.spatial = %d, this->targetLayers.temporal = %d", this->targetLayers.spatial, this->targetLayers.temporal); + this->targetLayers.spatial = newTargetSpatialLayer; this->targetLayers.temporal = newTargetTemporalLayer; From 0f3d127cde4c69a40f1e58f4b7563341464180ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Wed, 11 Feb 2026 11:39:37 +0100 Subject: [PATCH 07/10] adapt Rust tests --- rust/tests/integration/consumer.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/rust/tests/integration/consumer.rs b/rust/tests/integration/consumer.rs index 3dbd4e51ce..f959cdba1e 100644 --- a/rust/tests/integration/consumer.rs +++ b/rust/tests/integration/consumer.rs @@ -474,7 +474,13 @@ fn consume_succeeds() { producer_scores: vec![0] } ); - assert_eq!(audio_consumer.preferred_layers(), None); + assert_eq!( + audio_consumer.preferred_layers(), + Some(ConsumerLayers { + spatial_layer: 0, + temporal_layer: Some(0) + }) + ); assert_eq!(audio_consumer.current_layers(), None); assert_eq!( audio_consumer @@ -1320,7 +1326,13 @@ fn set_preferred_layers_succeeds() { .await .expect("Failed to set preferred layers consumer"); - assert_eq!(audio_consumer.preferred_layers(), None); + assert_eq!( + audio_consumer.preferred_layers(), + Some(ConsumerLayers { + spatial_layer: 0, + temporal_layer: Some(0) + }) + ); } { From 1c9c3bb984f0c79e9d6731d2334df358b5b9afac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Wed, 11 Feb 2026 12:44:57 +0100 Subject: [PATCH 08/10] cosmetic better tmp debug logs --- worker/src/RTC/MultiStreamConsumer.cpp | 251 +++++++++++++++++-------- 1 file changed, 177 insertions(+), 74 deletions(-) diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp index 36adc832af..e8b0fc3921 100644 --- a/worker/src/RTC/MultiStreamConsumer.cpp +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -368,7 +368,10 @@ namespace RTC if (!IsActive()) { // TODO: REMOVE - MS_DUMP("------ calling UpdateTargetLayers(-1, -1)"); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- all Producers are inactive, calling UpdateTargetLayers(-1, -1)"); + } UpdateTargetLayers(-1, -1); } @@ -667,7 +670,13 @@ namespace RTC if (provisionalTargetLayers != this->targetLayers) { // TODO: REMOVE - MS_DUMP("------ calling UpdateTargetLayers(%d, %d)", provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- calling UpdateTargetLayers(%d, %d)", + provisionalTargetLayers.spatial, + provisionalTargetLayers.temporal); + } UpdateTargetLayers(provisionalTargetLayers.spatial, provisionalTargetLayers.temporal); @@ -745,7 +754,7 @@ namespace RTC // TODO: REMOVE if (this->kind == RTC::Media::Kind::AUDIO) { - MS_DUMP("----- 1 audio packet | ssrc:%u", packet->GetSsrc()); + MS_DUMP("***** BEGIN audio packet | seq:%u", packet->GetSequenceNumber()); } #ifdef MS_RTC_LOGGER_RTP @@ -756,9 +765,6 @@ namespace RTC if (!IsActive()) { - // TODO: REMOVE - MS_DUMP("----------- NOT ACTIVE !!!!"); - // Only drop the packet in the RTP sequence manager if it belongs to the // current spatial layer. if (spatialLayer == this->currentSpatialLayer) @@ -767,16 +773,27 @@ namespace RTC packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::CONSUMER_INACTIVE); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- audio packet dropped CONSUMER_INACTIVE | seq:%u", packet->GetSequenceNumber()); + } + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); } - return; - } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO && spatialLayer != this->currentSpatialLayer) + { + MS_DUMP( + "----- !IsActive() => audio packet ignored because spatialLayer %d != this->currentSpatialLayer %d | seq:%u", + spatialLayer, + this->currentSpatialLayer, + packet->GetSequenceNumber()); + } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 2 audio packet | ssrc:%u", packet->GetSsrc()); + return; } if (this->targetLayers.temporal == -1) @@ -785,30 +802,33 @@ namespace RTC // current spatial layer. if (spatialLayer == this->currentSpatialLayer) { - MS_DUMP("----- INVALID_TARGET_LAYER audio packet | spatialLayer (%u) == this->currentSpatialLayer (%u)", spatialLayer, this->currentSpatialLayer); - #ifdef MS_RTC_LOGGER_RTP packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::INVALID_TARGET_LAYER); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- audio packet dropped INVALID_TARGET_LAYER | seq:%u", packet->GetSequenceNumber()); + } + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); } // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) + if (this->kind == RTC::Media::Kind::AUDIO && spatialLayer != this->currentSpatialLayer) { - MS_DUMP("----- DROPPED audio packet | this->targetLayers.temporal == -1"); + MS_DUMP( + "----- this->targetLayers.temporal == -1 => audio packet ignored because spatialLayer %d != this->currentSpatialLayer %d | seq:%u", + spatialLayer, + this->currentSpatialLayer, + packet->GetSequenceNumber()); } return; } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 3 audio packet | ssrc:%u", packet->GetSsrc()); - } - auto payloadType = packet->GetPayloadType(); // NOTE: This may happen if this Consumer supports just some codecs of those @@ -825,16 +845,28 @@ namespace RTC packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::UNSUPPORTED_PAYLOAD_TYPE); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- audio packet dropped UNSUPPORTED_PAYLOAD_TYPE | seq:%u", + packet->GetSequenceNumber()); + } + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); } - return; - } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO && spatialLayer != this->currentSpatialLayer) + { + MS_DUMP( + "----- unsupported codec => audio packet ignored because spatialLayer %d != this->currentSpatialLayer %d | seq:%u", + spatialLayer, + this->currentSpatialLayer, + packet->GetSequenceNumber()); + } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 4 audio packet | ssrc:%u", packet->GetSsrc()); + return; } bool shouldSwitchCurrentSpatialLayer{ false }; @@ -852,6 +884,12 @@ namespace RTC packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- audio packet dropped NOT_A_KEYFRAME | seq:%u", packet->GetSequenceNumber()); + } + // NOTE: Don't drop the packet in the RTP sequence manager since this // packet doesn't belong to the current spatial layer. @@ -875,13 +913,17 @@ namespace RTC // NOTE: Don't drop the packet in the RTP sequence manager since this // packet doesn't belong to the current spatial layer. - return; - } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- audio packet ignored (spatialLayer %d != this->currentSpatialLayer %d) | seq:%u", + spatialLayer, + this->currentSpatialLayer, + packet->GetSequenceNumber()); + } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 5 audio packet | ssrc:%u", packet->GetSsrc()); + return; } // If we need to sync and this is not a key frame, ignore the packet. @@ -893,6 +935,12 @@ namespace RTC packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::NOT_A_KEYFRAME); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- audio packet dropped NOT_A_KEYFRAME | seq:%u", packet->GetSequenceNumber()); + } + // NOTE: No need to drop the packet in the RTP sequence manager since here // we are blocking all packets but the key frame that would trigger sync // below. @@ -904,12 +952,6 @@ namespace RTC return; } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 6 audio packet | ssrc:%u", packet->GetSsrc()); - } - // Packets with only padding are not forwarded. if (packet->GetPayloadLength() == 0) { @@ -921,16 +963,26 @@ namespace RTC packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::EMPTY_PAYLOAD); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- audio packet dropped EMPTY_PAYLOAD | seq:%u", packet->GetSequenceNumber()); + } + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); } - return; - } + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO && spatialLayer != this->currentSpatialLayer) + { + MS_DUMP( + "----- unsupported codec => audio packet ignored because spatialLayer %d != this->currentSpatialLayer %d | seq:%u", + spatialLayer, + this->currentSpatialLayer, + packet->GetSequenceNumber()); + } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 7 audio packet | ssrc:%u", packet->GetSsrc()); + return; } // Whether this is the first packet after re-sync. @@ -1055,6 +1107,14 @@ namespace RTC RTC::RtcLogger::RtpPacket::DiscardReason::TOO_HIGH_TIMESTAMP_EXTRA_NEEDED); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- audio packet dropped TOO_HIGH_TIMESTAMP_EXTRA_NEEDED | seq:%u", + packet->GetSequenceNumber()); + } + // NOTE: Don't drop the packet in the RTP sequence manager since this // packet doesn't belong to the current spatial layer. @@ -1095,12 +1155,6 @@ namespace RTC this->keyFrameForTsOffsetRequested = false; } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 8 audio packet | ssrc:%u", packet->GetSsrc()); - } - if (!shouldSwitchCurrentSpatialLayer && this->checkingForOldPacketsInSpatialLayer) { // If this is a packet previous to the spatial layer switch, ignore the @@ -1115,6 +1169,14 @@ namespace RTC RTC::RtcLogger::RtpPacket::DiscardReason::PACKET_PREVIOUS_TO_SPATIAL_LAYER_SWITCH); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- audio packet dropped PACKET_PREVIOUS_TO_SPATIAL_LAYER_SWITCH | seq:%u", + packet->GetSequenceNumber()); + } + this->rtpSeqManager.Drop(packet->GetSequenceNumber()); return; @@ -1126,18 +1188,19 @@ namespace RTC } } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 9 audio packet | ssrc:%u", packet->GetSsrc()); - } - bool marker{ false }; if (shouldSwitchCurrentSpatialLayer) { // TODO: REMOVE - MS_DUMP("------ setting this->currentSpatialLayer = %d [previous value:%d]", this->targetLayers.spatial, this->currentSpatialLayer); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- setting this->currentSpatialLayer = %d [previous value:%d] | seq:%u", + this->targetLayers.spatial, + this->currentSpatialLayer, + packet->GetSequenceNumber()); + } // Update current spatial layer. this->currentSpatialLayer = this->targetLayers.spatial; @@ -1179,6 +1242,11 @@ namespace RTC #ifdef MS_RTC_LOGGER_RTP packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::DROPPED_BY_CODEC); #endif + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- audio packet dropped DROPPED_BY_CODEC | seq:%u", packet->GetSequenceNumber()); + } this->rtpSeqManager.Drop(packet->GetSequenceNumber()); @@ -1191,12 +1259,6 @@ namespace RTC } } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("----- 10 audio packet | ssrc:%u", packet->GetSsrc()); - } - // Update RTP seq number and timestamp based on NTP offset. uint16_t seq; const uint32_t timestamp = packet->GetTimestamp() - this->tsOffset; @@ -1245,7 +1307,7 @@ namespace RTC // TODO: REMOVE if (this->kind == RTC::Media::Kind::AUDIO) { - MS_DUMP("----- SENT! audio packet | ssrc:%u", packet->GetSsrc()); + MS_DUMP("+++++ AUDIO PACKET SENT :) | seq:%u", origSeq); } // Send the packet. @@ -1270,6 +1332,12 @@ namespace RTC #ifdef MS_RTC_LOGGER_RTP packet->logger.Discarded(RTC::RtcLogger::RtpPacket::DiscardReason::SEND_RTP_STREAM_DISCARDED); #endif + + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- audio packet dropped SEND_RTP_STREAM_DISCARDED | seq:%u", origSeq); + } } // Restore packet fields. @@ -1486,7 +1554,12 @@ namespace RTC this->targetLayerRetransmissionBuffer.clear(); // TODO: REMOVE - MS_DUMP("------ calling UpdateTargetLayers(-1, -1) [current this->currentSpatialLayer:%d]", this->currentSpatialLayer); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- calling UpdateTargetLayers(-1, -1) [current this->currentSpatialLayer:%d]", + this->currentSpatialLayer); + } UpdateTargetLayers(-1, -1); } @@ -1501,7 +1574,12 @@ namespace RTC this->targetLayerRetransmissionBuffer.clear(); // TODO: REMOVE - MS_DUMP("------ calling UpdateTargetLayers(-1, -1) [current this->currentSpatialLayer:%d]", this->currentSpatialLayer); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- calling UpdateTargetLayers(-1, -1) [current this->currentSpatialLayer:%d]", + this->currentSpatialLayer); + } UpdateTargetLayers(-1, -1); @@ -1702,7 +1780,13 @@ namespace RTC else { // TODO: REMOVE - MS_DUMP("------ calling UpdateTargetLayers(%d, %d)", newTargetLayers.spatial, newTargetLayers.temporal); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- calling UpdateTargetLayers(%d, %d)", + newTargetLayers.spatial, + newTargetLayers.temporal); + } UpdateTargetLayers(newTargetLayers.spatial, newTargetLayers.temporal); } @@ -1789,6 +1873,11 @@ namespace RTC { MS_TRACE(); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("***** UpdateTargetLayers(%d, %d)", newTargetSpatialLayer, newTargetTemporalLayer); + } + // If we don't have yet a RTP timestamp reference, set it now. if ( newTargetSpatialLayer != -1 && (this->tsReferenceSpatialLayer == -1 || @@ -1810,7 +1899,11 @@ namespace RTC if (newTargetSpatialLayer == -1) { // TODO: REMOVE - MS_DUMP("------ setting this->targetLayers.spatial = -1, this->targetLayers.temporal = -1, this->currentSpatialLayer = -1"); + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- newTargetSpatialLayer == -1 => setting this->targetLayers.spatial = -1, this->targetLayers.temporal = -1, this->currentSpatialLayer = -1"); + } // Unset current and target layers. this->targetLayers.spatial = -1; @@ -1831,8 +1924,14 @@ namespace RTC return; } - // TODO - MS_DUMP("---- setting this->targetLayers.spatial = %d, this->targetLayers.temporal = %d", this->targetLayers.spatial, this->targetLayers.temporal); + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- setting this->targetLayers.spatial = %d, this->targetLayers.temporal = %d", + newTargetSpatialLayer, + newTargetTemporalLayer); + } this->targetLayers.spatial = newTargetSpatialLayer; this->targetLayers.temporal = newTargetTemporalLayer; @@ -1965,8 +2064,12 @@ namespace RTC if (this->currentSpatialLayer == -1) { - // TODO: Remove - MS_DUMP("------ this->currentSpatialLayer == -1 => nullptr"); + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- this->currentSpatialLayer == -1 => nullptr"); + } + return nullptr; } From 89df2d43c207882827cbd7f5bf8310e94580e3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Wed, 11 Feb 2026 15:23:26 +0100 Subject: [PATCH 09/10] more --- worker/include/RTC/ConsumerTypes.hpp | 3 +++ worker/meson.build | 1 + worker/src/RTC/ConsumerTypes.cpp | 21 +++++++++++++++++++++ worker/src/RTC/MultiStreamConsumer.cpp | 12 ++++++++++++ 4 files changed, 37 insertions(+) create mode 100644 worker/src/RTC/ConsumerTypes.cpp diff --git a/worker/include/RTC/ConsumerTypes.hpp b/worker/include/RTC/ConsumerTypes.hpp index 0544167cbd..54f6ffd853 100644 --- a/worker/include/RTC/ConsumerTypes.hpp +++ b/worker/include/RTC/ConsumerTypes.hpp @@ -19,6 +19,7 @@ namespace RTC } VideoLayers(const VideoLayers& other) = default; + bool operator==(const VideoLayers& other) const { return spatial == other.spatial && temporal == other.temporal; @@ -29,6 +30,8 @@ namespace RTC return !(*this == other); } + void Dump(int indentation) const; + void Reset() { spatial = -1; diff --git a/worker/meson.build b/worker/meson.build index ec1d3fdd52..ad0cf99aed 100644 --- a/worker/meson.build +++ b/worker/meson.build @@ -104,6 +104,7 @@ common_sources = [ 'src/RTC/ActiveSpeakerObserver.cpp', 'src/RTC/AudioLevelObserver.cpp', 'src/RTC/Consumer.cpp', + 'src/RTC/ConsumerTypes.cpp', 'src/RTC/DataConsumer.cpp', 'src/RTC/DataProducer.cpp', 'src/RTC/DirectTransport.cpp', diff --git a/worker/src/RTC/ConsumerTypes.cpp b/worker/src/RTC/ConsumerTypes.cpp new file mode 100644 index 0000000000..f465359e76 --- /dev/null +++ b/worker/src/RTC/ConsumerTypes.cpp @@ -0,0 +1,21 @@ +#define MS_CLASS "RTC::ConsumerTypes" +// #define MS_LOG_DEV_LEVEL 3 + +#include "RTC/ConsumerTypes.hpp" +#include "Logger.hpp" + +namespace RTC +{ + namespace ConsumerTypes + { + void VideoLayers::Dump(int indentation) const + { + MS_TRACE(); + + MS_DUMP_CLEAN(indentation, ""); + MS_DUMP_CLEAN( + indentation, " spatial:%" PRIi16 ", temporal:%" PRIi16, this->spatial, this->temporal); + MS_DUMP_CLEAN(indentation, ""); + } + } // namespace ConsumerTypes +} // namespace RTC diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp index e8b0fc3921..800e2176c1 100644 --- a/worker/src/RTC/MultiStreamConsumer.cpp +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -755,6 +755,12 @@ namespace RTC if (this->kind == RTC::Media::Kind::AUDIO) { MS_DUMP("***** BEGIN audio packet | seq:%u", packet->GetSequenceNumber()); + MS_DUMP("---- this->targetLayers:"); + this->targetLayers.Dump(6); + MS_DUMP("---- this->provisionalTargetLayers:"); + this->provisionalTargetLayers.Dump(6); + MS_DUMP("---- this->targetLayers:"); + this->targetLayers.Dump(6); } #ifdef MS_RTC_LOGGER_RTP @@ -1308,6 +1314,12 @@ namespace RTC if (this->kind == RTC::Media::Kind::AUDIO) { MS_DUMP("+++++ AUDIO PACKET SENT :) | seq:%u", origSeq); + MS_DUMP("---- this->targetLayers:"); + this->targetLayers.Dump(6); + MS_DUMP("---- this->provisionalTargetLayers:"); + this->provisionalTargetLayers.Dump(6); + MS_DUMP("---- this->targetLayers:"); + this->targetLayers.Dump(6); } // Send the packet. From 212a2eeb2ef2433f2975536341f42874bbdbdc9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Wed, 11 Feb 2026 16:03:02 +0100 Subject: [PATCH 10/10] fix audio Consumer issue --- worker/include/RTC/Consumer.hpp | 2 +- worker/include/RTC/MultiStreamConsumer.hpp | 8 +++ worker/src/RTC/MultiStreamConsumer.cpp | 60 +++++++++++----------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/worker/include/RTC/Consumer.hpp b/worker/include/RTC/Consumer.hpp index 86f6333342..a16c325cc2 100644 --- a/worker/include/RTC/Consumer.hpp +++ b/worker/include/RTC/Consumer.hpp @@ -133,7 +133,7 @@ namespace RTC RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) = 0; virtual void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) = 0; void ProducerClosed(); - void SetExternallyManagedBitrate() + virtual void SetExternallyManagedBitrate() { this->externallyManagedBitrate = true; } diff --git a/worker/include/RTC/MultiStreamConsumer.hpp b/worker/include/RTC/MultiStreamConsumer.hpp index 9d91d2c15f..c23ee2d0a3 100644 --- a/worker/include/RTC/MultiStreamConsumer.hpp +++ b/worker/include/RTC/MultiStreamConsumer.hpp @@ -61,6 +61,14 @@ namespace RTC void ProducerRtpStreamScore( RTC::RTP::RtpStreamRecv* rtpStream, uint8_t score, uint8_t previousScore) override; void ProducerRtcpSenderReport(RTC::RTP::RtpStreamRecv* rtpStream, bool first) override; + void SetExternallyManagedBitrate() override + { + // Only allow externally managed bitrate video. + if (this->kind == RTC::Media::Kind::VIDEO) + { + this->externallyManagedBitrate = true; + } + } uint8_t GetBitratePriority() const override; uint32_t IncreaseLayer(uint32_t bitrate, bool considerLoss) override; void ApplyLayers() override; diff --git a/worker/src/RTC/MultiStreamConsumer.cpp b/worker/src/RTC/MultiStreamConsumer.cpp index 800e2176c1..6917280249 100644 --- a/worker/src/RTC/MultiStreamConsumer.cpp +++ b/worker/src/RTC/MultiStreamConsumer.cpp @@ -414,10 +414,16 @@ namespace RTC { MS_TRACE(); + // Only for video. + if (this->kind == RTC::Media::Kind::AUDIO) + { + return 0u; + } + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); // Audio SimpleConsumer does not play the BWE game. - if (this->kind != RTC::Media::Kind::VIDEO) + if (this->kind == RTC::Media::Kind::AUDIO && this->type == RTC::RtpParameters::Type::SIMPLE) { return 0u; } @@ -702,14 +708,14 @@ namespace RTC { MS_TRACE(); - MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); - - // Audio SimpleConsumer does not play the BWE game. - if (this->kind != RTC::Media::Kind::VIDEO) + // Only for video. + if (this->kind == RTC::Media::Kind::AUDIO) { return 0u; } + MS_ASSERT(this->externallyManagedBitrate, "bitrate is not externally managed"); + if (!IsActive()) { return 0u; @@ -751,18 +757,6 @@ namespace RTC { MS_TRACE(); - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("***** BEGIN audio packet | seq:%u", packet->GetSequenceNumber()); - MS_DUMP("---- this->targetLayers:"); - this->targetLayers.Dump(6); - MS_DUMP("---- this->provisionalTargetLayers:"); - this->provisionalTargetLayers.Dump(6); - MS_DUMP("---- this->targetLayers:"); - this->targetLayers.Dump(6); - } - #ifdef MS_RTC_LOGGER_RTP packet->logger.consumerId = this->id; #endif @@ -1310,18 +1304,6 @@ namespace RTC this->lastSentPacketHasMarker = packet->HasMarker(); } - // TODO: REMOVE - if (this->kind == RTC::Media::Kind::AUDIO) - { - MS_DUMP("+++++ AUDIO PACKET SENT :) | seq:%u", origSeq); - MS_DUMP("---- this->targetLayers:"); - this->targetLayers.Dump(6); - MS_DUMP("---- this->provisionalTargetLayers:"); - this->provisionalTargetLayers.Dump(6); - MS_DUMP("---- this->targetLayers:"); - this->targetLayers.Dump(6); - } - // Send the packet. this->listener->OnConsumerSendRtpPacket(this, packet); @@ -1773,6 +1755,12 @@ namespace RTC { MS_TRACE(); + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("***** MayChangeLayers(force:%s)", force ? "true" : "false"); + } + RTC::ConsumerTypes::VideoLayers newTargetLayers; if (RecalculateTargetLayers(newTargetLayers)) @@ -1784,8 +1772,21 @@ namespace RTC // will let us change it when it considers. if (this->externallyManagedBitrate) { + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP("----- this->externallyManagedBitrate == true"); + } + if (newTargetLayers.spatial != this->targetLayers.spatial || force) { + // TODO: REMOVE + if (this->kind == RTC::Media::Kind::AUDIO) + { + MS_DUMP( + "----- this->externallyManagedBitrate == true => calling listener->OnConsumerNeedBitrateChange()"); + } + this->listener->OnConsumerNeedBitrateChange(this); } } @@ -1885,6 +1886,7 @@ namespace RTC { MS_TRACE(); + // TODO: REMOVE if (this->kind == RTC::Media::Kind::AUDIO) { MS_DUMP("***** UpdateTargetLayers(%d, %d)", newTargetSpatialLayer, newTargetTemporalLayer);