#include "MediaManager.h" #include "absl/strings/match.h" #include "api/audio_codecs/audio_decoder_factory_template.h" #include "api/audio_codecs/audio_encoder_factory_template.h" #include "api/audio_codecs/opus/audio_decoder_opus.h" #include "api/audio_codecs/opus/audio_encoder_opus.h" #include "api/rtp_parameters.h" #include "api/task_queue/default_task_queue_factory.h" #include "media/base/codec.h" #include "media/base/media_constants.h" #include "media/engine/webrtc_media_engine.h" #include "modules/audio_device/include/audio_device_default.h" #include "rtc_base/task_utils/repeating_task.h" #include "system_wrappers/include/field_trial.h" #include "api/video/builtin_video_bitrate_allocator_factory.h" #include "api/video/video_bitrate_allocation.h" #include "call/call.h" #if TARGET_OS_IPHONE #include "CodecsApple.h" #else #error "Unsupported platform" #endif #ifdef TGVOIP_NAMESPACE namespace TGVOIP_NAMESPACE { #endif static const uint32_t ssrcAudioIncoming = 1; static const uint32_t ssrcAudioOutgoing = 2; static const uint32_t ssrcAudioFecIncoming = 5; static const uint32_t ssrcAudioFecOutgoing = 6; static const uint32_t ssrcVideoIncoming = 3; static const uint32_t ssrcVideoOutgoing = 4; static const uint32_t ssrcVideoFecIncoming = 7; static const uint32_t ssrcVideoFecOutgoing = 8; static void AddDefaultFeedbackParams(cricket::VideoCodec *codec) { // Don't add any feedback params for RED and ULPFEC. if (codec->name == cricket::kRedCodecName || codec->name == cricket::kUlpfecCodecName) return; codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamRemb, cricket::kParamValueEmpty)); codec->AddFeedbackParam( cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc, cricket::kParamValueEmpty)); // Don't add any more feedback params for FLEXFEC. if (codec->name == cricket::kFlexfecCodecName) return; codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamCcm, cricket::kRtcpFbCcmParamFir)); codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kParamValueEmpty)); codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kRtcpFbNackParamPli)); if (codec->name == cricket::kVp8CodecName && webrtc::field_trial::IsEnabled("WebRTC-RtcpLossNotification")) { codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamLntf, cricket::kParamValueEmpty)); } } static std::vector AssignPayloadTypesAndDefaultCodecs(std::vector input_formats) { if (input_formats.empty()) return std::vector(); static const int kFirstDynamicPayloadType = 96; static const int kLastDynamicPayloadType = 127; int payload_type = kFirstDynamicPayloadType; input_formats.push_back(webrtc::SdpVideoFormat(cricket::kRedCodecName)); input_formats.push_back(webrtc::SdpVideoFormat(cricket::kUlpfecCodecName)); if (true) { webrtc::SdpVideoFormat flexfec_format(cricket::kFlexfecCodecName); // This value is currently arbitrarily set to 10 seconds. (The unit // is microseconds.) This parameter MUST be present in the SDP, but // we never use the actual value anywhere in our code however. // TODO(brandtr): Consider honouring this value in the sender and receiver. flexfec_format.parameters = {{cricket::kFlexfecFmtpRepairWindow, "10000000"}}; input_formats.push_back(flexfec_format); } std::vector output_codecs; for (const webrtc::SdpVideoFormat& format : input_formats) { cricket::VideoCodec codec(format); codec.id = payload_type; AddDefaultFeedbackParams(&codec); output_codecs.push_back(codec); // Increment payload type. ++payload_type; if (payload_type > kLastDynamicPayloadType) { RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest."; break; } // Add associated RTX codec for non-FEC codecs. if (!absl::EqualsIgnoreCase(codec.name, cricket::kUlpfecCodecName) && !absl::EqualsIgnoreCase(codec.name, cricket::kFlexfecCodecName)) { output_codecs.push_back( cricket::VideoCodec::CreateRtxCodec(payload_type, codec.id)); // Increment payload type. ++payload_type; if (payload_type > kLastDynamicPayloadType) { RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest."; break; } } } return output_codecs; } static absl::optional selectVideoCodec(std::vector &codecs) { bool useVP9 = false; bool useH265 = true; for (auto &codec : codecs) { if (useVP9) { if (codec.name == cricket::kVp9CodecName) { return absl::optional(codec); } } else if (useH265) { if (codec.name == cricket::kH265CodecName) { return absl::optional(codec); } } else { if (codec.name == cricket::kH264CodecName) { return absl::optional(codec); } } } return absl::optional(); } static rtc::Thread *makeWorkerThread() { static std::unique_ptr value = rtc::Thread::Create(); value->SetName("WebRTC-Worker", nullptr); value->Start(); return value.get(); } static rtc::Thread *getWorkerThread() { static rtc::Thread *value = makeWorkerThread(); return value; } MediaManager::MediaManager( rtc::Thread *thread, bool isOutgoing, std::function packetEmitted ) : _packetEmitted(packetEmitted), _thread(thread), _eventLog(std::make_unique()), _taskQueueFactory(webrtc::CreateDefaultTaskQueueFactory()) { _ssrcAudio.incoming = isOutgoing ? ssrcAudioIncoming : ssrcAudioOutgoing; _ssrcAudio.outgoing = (!isOutgoing) ? ssrcAudioIncoming : ssrcAudioOutgoing; _ssrcAudio.fecIncoming = isOutgoing ? ssrcAudioFecIncoming : ssrcAudioFecOutgoing; _ssrcAudio.fecOutgoing = (!isOutgoing) ? ssrcAudioFecIncoming : ssrcAudioFecOutgoing; _ssrcVideo.incoming = isOutgoing ? ssrcVideoIncoming : ssrcVideoOutgoing; _ssrcVideo.outgoing = (!isOutgoing) ? ssrcVideoIncoming : ssrcVideoOutgoing; _ssrcVideo.fecIncoming = isOutgoing ? ssrcVideoFecIncoming : ssrcVideoFecOutgoing; _ssrcVideo.fecOutgoing = (!isOutgoing) ? ssrcVideoFecIncoming : ssrcVideoFecOutgoing; _audioNetworkInterface = std::unique_ptr(new MediaManager::NetworkInterfaceImpl(this, false)); _videoNetworkInterface = std::unique_ptr(new MediaManager::NetworkInterfaceImpl(this, true)); webrtc::field_trial::InitFieldTrialsFromString( "WebRTC-Audio-SendSideBwe/Enabled/" "WebRTC-Audio-Allocation/min:6kbps,max:32kbps/" "WebRTC-Audio-OpusMinPacketLossRate/Enabled-1/" "WebRTC-FlexFEC-03/Enabled/" "WebRTC-FlexFEC-03-Advertised/Enabled/" ); configurePlatformAudio(); _videoBitrateAllocatorFactory = webrtc::CreateBuiltinVideoBitrateAllocatorFactory(); cricket::MediaEngineDependencies mediaDeps; mediaDeps.task_queue_factory = _taskQueueFactory.get(); mediaDeps.audio_encoder_factory = webrtc::CreateAudioEncoderFactory(); mediaDeps.audio_decoder_factory = webrtc::CreateAudioDecoderFactory(); auto videoEncoderFactory = makeVideoEncoderFactory(); std::vector videoCodecs = AssignPayloadTypesAndDefaultCodecs(videoEncoderFactory->GetSupportedFormats()); mediaDeps.video_encoder_factory = makeVideoEncoderFactory(); mediaDeps.video_decoder_factory = makeVideoDecoderFactory(); mediaDeps.audio_processing = webrtc::AudioProcessingBuilder().Create(); _mediaEngine = cricket::CreateMediaEngine(std::move(mediaDeps)); _mediaEngine->Init(); webrtc::Call::Config callConfig(_eventLog.get()); callConfig.task_queue_factory = _taskQueueFactory.get(); callConfig.trials = &_fieldTrials; callConfig.audio_state = _mediaEngine->voice().GetAudioState(); _call.reset(webrtc::Call::Create(callConfig)); _audioChannel.reset(_mediaEngine->voice().CreateMediaChannel(_call.get(), cricket::MediaConfig(), cricket::AudioOptions(), webrtc::CryptoOptions::NoGcm())); _videoChannel.reset(_mediaEngine->video().CreateMediaChannel(_call.get(), cricket::MediaConfig(), cricket::VideoOptions(), webrtc::CryptoOptions::NoGcm(), _videoBitrateAllocatorFactory.get())); _audioChannel->AddSendStream(cricket::StreamParams::CreateLegacy(_ssrcAudio.outgoing)); const uint32_t opusClockrate = 48000; const uint16_t opusSdpPayload = 111; const char *opusSdpName = "opus"; const uint8_t opusSdpChannels = 2; const uint32_t opusSdpBitrate = 0; const uint8_t opusMinBitrateKbps = 6; const uint8_t opusMaxBitrateKbps = 32; const uint8_t opusStartBitrateKbps = 6; const uint8_t opusPTimeMs = 120; const int extensionSequenceOne = 1; cricket::AudioCodec opusCodec(opusSdpPayload, opusSdpName, opusClockrate, opusSdpBitrate, opusSdpChannels); opusCodec.AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc)); opusCodec.SetParam(cricket::kCodecParamMinBitrate, opusMinBitrateKbps); opusCodec.SetParam(cricket::kCodecParamStartBitrate, opusStartBitrateKbps); opusCodec.SetParam(cricket::kCodecParamMaxBitrate, opusMaxBitrateKbps); opusCodec.SetParam(cricket::kCodecParamUseInbandFec, 1); opusCodec.SetParam(cricket::kCodecParamPTime, opusPTimeMs); cricket::AudioSendParameters audioSendPrameters; audioSendPrameters.codecs.push_back(opusCodec); audioSendPrameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, extensionSequenceOne); audioSendPrameters.options.echo_cancellation = false; //audioSendPrameters.options.experimental_ns = false; audioSendPrameters.options.noise_suppression = false; audioSendPrameters.options.auto_gain_control = false; audioSendPrameters.options.highpass_filter = false; audioSendPrameters.options.typing_detection = false; //audioSendPrameters.max_bandwidth_bps = 16000; audioSendPrameters.rtcp.reduced_size = true; audioSendPrameters.rtcp.remote_estimate = true; _audioChannel->SetSendParameters(audioSendPrameters); _audioChannel->SetInterface(_audioNetworkInterface.get(), webrtc::MediaTransportConfig()); cricket::AudioRecvParameters audioRecvParameters; audioRecvParameters.codecs.emplace_back(opusSdpPayload, opusSdpName, opusClockrate, opusSdpBitrate, opusSdpChannels); audioRecvParameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, extensionSequenceOne); audioRecvParameters.rtcp.reduced_size = true; audioRecvParameters.rtcp.remote_estimate = true; _audioChannel->SetRecvParameters(audioRecvParameters); _audioChannel->AddRecvStream(cricket::StreamParams::CreateLegacy(_ssrcAudio.incoming)); _audioChannel->SetPlayout(true); cricket::StreamParams sp; cricket::SsrcGroup sg(cricket::kFecFrSsrcGroupSemantics, {_ssrcVideo.outgoing, _ssrcVideo.fecOutgoing}); sp.ssrcs = {_ssrcVideo.outgoing}; sp.ssrc_groups.push_back(sg); sp.cname = "cname"; _videoChannel->AddSendStream(sp); auto videoCodec = selectVideoCodec(videoCodecs); if (videoCodec.has_value()) { _nativeVideoSource = makeVideoSource(_thread, getWorkerThread()); auto codec = videoCodec.value(); codec.SetParam(cricket::kCodecParamMinBitrate, 64); codec.SetParam(cricket::kCodecParamStartBitrate, 512); codec.SetParam(cricket::kCodecParamMaxBitrate, 2500); _videoCapturer = makeVideoCapturer(_nativeVideoSource); cricket::VideoSendParameters videoSendParameters; videoSendParameters.codecs.push_back(codec); for (auto &c : videoCodecs) { if (c.name == cricket::kFlexfecCodecName) { videoSendParameters.codecs.push_back(c); break; } } videoSendParameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, extensionSequenceOne); //send_parameters.max_bandwidth_bps = 800000; //send_parameters.rtcp.reduced_size = true; //videoSendParameters.rtcp.remote_estimate = true; _videoChannel->SetSendParameters(videoSendParameters); _videoChannel->SetVideoSend(_ssrcVideo.outgoing, NULL, _nativeVideoSource.get()); _videoChannel->SetVideoSend(_ssrcVideo.fecOutgoing, NULL, nullptr); _videoChannel->SetInterface(_videoNetworkInterface.get(), webrtc::MediaTransportConfig()); cricket::VideoRecvParameters videoRecvParameters; videoRecvParameters.codecs.emplace_back(codec); for (auto &c : videoCodecs) { if (c.name == cricket::kFlexfecCodecName) { videoRecvParameters.codecs.push_back(c); break; } } videoRecvParameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, extensionSequenceOne); //recv_parameters.rtcp.reduced_size = true; videoRecvParameters.rtcp.remote_estimate = true; cricket::StreamParams videoRecvStreamParams; cricket::SsrcGroup videoRecvSsrcGroup(cricket::kFecFrSsrcGroupSemantics, {_ssrcVideo.incoming, _ssrcVideo.fecIncoming}); videoRecvStreamParams.ssrcs = {_ssrcVideo.incoming}; videoRecvStreamParams.ssrc_groups.push_back(videoRecvSsrcGroup); videoRecvStreamParams.cname = "cname"; _videoChannel->AddRecvStream(videoRecvStreamParams); _videoChannel->SetRecvParameters(videoRecvParameters); /*webrtc::FlexfecReceiveStream::Config config(_videoNetworkInterface.get()); config.payload_type = 118; config.protected_media_ssrcs = {1324234}; webrtc::FlexfecReceiveStream* stream; std::list streams;*/ } } MediaManager::~MediaManager() { assert(_thread->IsCurrent()); _call->SignalChannelNetworkState(webrtc::MediaType::AUDIO, webrtc::kNetworkDown); _call->SignalChannelNetworkState(webrtc::MediaType::VIDEO, webrtc::kNetworkDown); _audioChannel->OnReadyToSend(false); _audioChannel->SetSend(false); _audioChannel->SetAudioSend(_ssrcAudio.outgoing, false, nullptr, &_audioSource); _audioChannel->SetPlayout(false); _audioChannel->RemoveRecvStream(_ssrcAudio.incoming); _audioChannel->RemoveSendStream(_ssrcAudio.outgoing); _audioChannel->SetInterface(nullptr, webrtc::MediaTransportConfig()); _videoChannel->RemoveRecvStream(_ssrcVideo.incoming); _videoChannel->RemoveRecvStream(_ssrcVideo.fecIncoming); _videoChannel->RemoveSendStream(_ssrcVideo.outgoing); _videoChannel->RemoveSendStream(_ssrcVideo.fecOutgoing); _videoChannel->SetVideoSend(_ssrcVideo.outgoing, NULL, nullptr); _videoChannel->SetVideoSend(_ssrcVideo.fecOutgoing, NULL, nullptr); _videoChannel->SetInterface(nullptr, webrtc::MediaTransportConfig()); } void MediaManager::setIsConnected(bool isConnected) { if (isConnected) { _call->SignalChannelNetworkState(webrtc::MediaType::AUDIO, webrtc::kNetworkUp); _call->SignalChannelNetworkState(webrtc::MediaType::VIDEO, webrtc::kNetworkUp); } else { _call->SignalChannelNetworkState(webrtc::MediaType::AUDIO, webrtc::kNetworkDown); _call->SignalChannelNetworkState(webrtc::MediaType::VIDEO, webrtc::kNetworkDown); } if (_audioChannel) { _audioChannel->OnReadyToSend(isConnected); _audioChannel->SetSend(isConnected); _audioChannel->SetAudioSend(_ssrcAudio.outgoing, isConnected, nullptr, &_audioSource); } if (_videoChannel) { _videoChannel->OnReadyToSend(isConnected); _videoChannel->SetSend(isConnected); } } void MediaManager::receivePacket(const rtc::CopyOnWriteBuffer &packet) { if (packet.size() < 1) { return; } uint8_t header = ((uint8_t *)packet.data())[0]; rtc::CopyOnWriteBuffer unwrappedPacket = packet.Slice(1, packet.size() - 1); if (header == 0xba) { if (_audioChannel) { _audioChannel->OnPacketReceived(unwrappedPacket, -1); } } else if (header == 0xbf) { if (_videoChannel) { _videoChannel->OnPacketReceived(unwrappedPacket, -1); } } } void MediaManager::notifyPacketSent(const rtc::SentPacket &sentPacket) { _call->OnSentPacket(sentPacket); } void MediaManager::setIncomingVideoOutput(std::shared_ptr> sink) { _currentIncomingVideoSink = sink; _videoChannel->SetSink(_ssrcVideo.incoming, sink.get()); } void MediaManager::setOutgoingVideoOutput(std::shared_ptr> sink) { _currentOutgoingVideoSink = sink; _nativeVideoSource->AddOrUpdateSink(sink.get(), rtc::VideoSinkWants()); } MediaManager::NetworkInterfaceImpl::NetworkInterfaceImpl(MediaManager *mediaManager, bool isVideo) : _mediaManager(mediaManager), _isVideo(isVideo) { } bool MediaManager::NetworkInterfaceImpl::SendPacket(rtc::CopyOnWriteBuffer *packet, const rtc::PacketOptions& options) { rtc::CopyOnWriteBuffer wrappedPacket; uint8_t header = _isVideo ? 0xbf : 0xba; wrappedPacket.AppendData(&header, 1); wrappedPacket.AppendData(*packet); _mediaManager->_packetEmitted(wrappedPacket); rtc::SentPacket sentPacket(options.packet_id, rtc::TimeMillis(), options.info_signaled_after_sent); _mediaManager->notifyPacketSent(sentPacket); return true; } bool MediaManager::NetworkInterfaceImpl::SendRtcp(rtc::CopyOnWriteBuffer *packet, const rtc::PacketOptions& options) { rtc::CopyOnWriteBuffer wrappedPacket; uint8_t header = _isVideo ? 0xbf : 0xba; wrappedPacket.AppendData(&header, 1); wrappedPacket.AppendData(*packet); _mediaManager->_packetEmitted(wrappedPacket); rtc::SentPacket sentPacket(options.packet_id, rtc::TimeMillis(), options.info_signaled_after_sent); _mediaManager->notifyPacketSent(sentPacket); return true; } int MediaManager::NetworkInterfaceImpl::SetOption(cricket::MediaChannel::NetworkInterface::SocketType, rtc::Socket::Option, int) { return -1; } #ifdef TGVOIP_NAMESPACE } #endif