diff --git a/submodules/TgVoipWebrtc/BUILD b/submodules/TgVoipWebrtc/BUILD index 199e7c4cfd..b4890a5b45 100644 --- a/submodules/TgVoipWebrtc/BUILD +++ b/submodules/TgVoipWebrtc/BUILD @@ -27,6 +27,8 @@ objc_library( "-Ithird-party/webrtc/webrtc-ios/src/sdk/objc", "-Ithird-party/webrtc/webrtc-ios/src/sdk/objc/base", "-Ithird-party/webrtc/webrtc-ios/src/sdk/objc/components/renderer/metal", + "-Ithird-party/webrtc/webrtc-ios/src/sdk/objc/components/video_codec", + "-Ithird-party/webrtc/webrtc-ios/src/third_party/libyuv/include", "-DWEBRTC_IOS", "-DWEBRTC_MAC", "-DWEBRTC_POSIX", diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoDecoderFactory.h b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoDecoderFactory.h new file mode 100644 index 0000000000..7de5cfa86d --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoDecoderFactory.h @@ -0,0 +1,25 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" +#import "RTCVideoDecoderFactory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** This decoder factory include support for all codecs bundled with WebRTC. If using custom + * codecs, create custom implementations of RTCVideoEncoderFactory and RTCVideoDecoderFactory. + */ +RTC_OBJC_EXPORT +@interface TGRTCDefaultVideoDecoderFactory : NSObject +@end + +NS_ASSUME_NONNULL_END diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoDecoderFactory.mm b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoDecoderFactory.mm new file mode 100644 index 0000000000..15f643fb2c --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoDecoderFactory.mm @@ -0,0 +1,90 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "TGRTCDefaultVideoDecoderFactory.h" + +#import "RTCH264ProfileLevelId.h" +#import "RTCVideoDecoderH264.h" +#import "api/video_codec/RTCVideoCodecConstants.h" +#import "api/video_codec/RTCVideoDecoderVP8.h" +#import "base/RTCVideoCodecInfo.h" +#if defined(RTC_ENABLE_VP9) +#import "api/video_codec/RTCVideoDecoderVP9.h" +#endif +#if !defined(DISABLE_H265) +#import "RTCH265ProfileLevelId.h" +#import "TGRTCVideoDecoderH265.h" +#endif + +@implementation TGRTCDefaultVideoDecoderFactory + +- (NSArray *)supportedCodecs { + NSDictionary *constrainedHighParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTCVideoCodecInfo *constrainedHighInfo = + [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedHighParams]; + + NSDictionary *constrainedBaselineParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTCVideoCodecInfo *constrainedBaselineInfo = + [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedBaselineParams]; + + RTCVideoCodecInfo *vp8Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp8Name]; + +#if defined(RTC_ENABLE_VP9) + RTCVideoCodecInfo *vp9Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp9Name]; +#endif + +#if !defined(DISABLE_H265) + RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name]; +#endif + + return @[ + constrainedHighInfo, + constrainedBaselineInfo, + vp8Info, +#if defined(RTC_ENABLE_VP9) + vp9Info, +#endif +#if !defined(DISABLE_H265) + h265Info, +#endif + ]; +} + +- (id)createDecoder:(RTCVideoCodecInfo *)info { + if ([info.name isEqualToString:kRTCVideoCodecH264Name]) { + return [[RTCVideoDecoderH264 alloc] init]; + } else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) { + return [RTCVideoDecoderVP8 vp8Decoder]; +#if defined(RTC_ENABLE_VP9) + } else if ([info.name isEqualToString:kRTCVideoCodecVp9Name]) { + return [RTCVideoDecoderVP9 vp9Decoder]; +#endif +#if !defined(DISABLE_H265) + } else if (@available(iOS 11, *)) { + if ([info.name isEqualToString:kRTCVideoCodecH265Name]) { + return [[TGRTCVideoDecoderH265 alloc] init]; + } +#endif + } + + return nil; +} + +@end diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoEncoderFactory.h b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoEncoderFactory.h new file mode 100644 index 0000000000..24c5b5980a --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoEncoderFactory.h @@ -0,0 +1,30 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" +#import "RTCVideoEncoderFactory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** This encoder factory include support for all codecs bundled with WebRTC. If using custom + * codecs, create custom implementations of RTCVideoEncoderFactory and RTCVideoDecoderFactory. + */ +RTC_OBJC_EXPORT +@interface TGRTCDefaultVideoEncoderFactory : NSObject + +@property(nonatomic, retain) RTCVideoCodecInfo *preferredCodec; + ++ (NSArray *)supportedCodecs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoEncoderFactory.mm b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoEncoderFactory.mm new file mode 100644 index 0000000000..7a12709539 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCDefaultVideoEncoderFactory.mm @@ -0,0 +1,106 @@ +/* + * Copyright 2017 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "TGRTCDefaultVideoEncoderFactory.h" + +#import "RTCH264ProfileLevelId.h" +#import "RTCVideoEncoderH264.h" +#import "api/video_codec/RTCVideoCodecConstants.h" +#import "api/video_codec/RTCVideoEncoderVP8.h" +#import "base/RTCVideoCodecInfo.h" +#if defined(RTC_ENABLE_VP9) +#import "api/video_codec/RTCVideoEncoderVP9.h" +#endif +#if !defined(DISABLE_H265) +#import "RTCH265ProfileLevelId.h" +#import "TGRTCVideoEncoderH265.h" +#endif + +@implementation TGRTCDefaultVideoEncoderFactory + +@synthesize preferredCodec; + ++ (NSArray *)supportedCodecs { + NSDictionary *constrainedHighParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedHigh, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTCVideoCodecInfo *constrainedHighInfo = + [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedHighParams]; + + NSDictionary *constrainedBaselineParams = @{ + @"profile-level-id" : kRTCMaxSupportedH264ProfileLevelConstrainedBaseline, + @"level-asymmetry-allowed" : @"1", + @"packetization-mode" : @"1", + }; + RTCVideoCodecInfo *constrainedBaselineInfo = + [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH264Name + parameters:constrainedBaselineParams]; + + RTCVideoCodecInfo *vp8Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp8Name]; + +#if defined(RTC_ENABLE_VP9) + RTCVideoCodecInfo *vp9Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecVp9Name]; +#endif + +#if !defined(DISABLE_H265) + RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name]; +#endif + + return @[ + constrainedHighInfo, + constrainedBaselineInfo, + vp8Info, +#if defined(RTC_ENABLE_VP9) + vp9Info, +#endif +#if !defined(DISABLE_H265) + h265Info, +#endif + ]; +} + +- (id)createEncoder:(RTCVideoCodecInfo *)info { + if ([info.name isEqualToString:kRTCVideoCodecH264Name]) { + return [[RTCVideoEncoderH264 alloc] initWithCodecInfo:info]; + } else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) { + return [RTCVideoEncoderVP8 vp8Encoder]; +#if defined(RTC_ENABLE_VP9) + } else if ([info.name isEqualToString:kRTCVideoCodecVp9Name]) { + return [RTCVideoEncoderVP9 vp9Encoder]; +#endif +#if !defined(DISABLE_H265) + } else if (@available(iOS 11, *)) { + if ([info.name isEqualToString:kRTCVideoCodecH265Name]) { + return [[TGRTCVideoEncoderH265 alloc] initWithCodecInfo:info]; + } +#endif + } + + return nil; +} + +- (NSArray *)supportedCodecs { + NSMutableArray *codecs = [[[self class] supportedCodecs] mutableCopy]; + + NSMutableArray *orderedCodecs = [NSMutableArray array]; + NSUInteger index = [codecs indexOfObject:self.preferredCodec]; + if (index != NSNotFound) { + [orderedCodecs addObject:[codecs objectAtIndex:index]]; + [codecs removeObjectAtIndex:index]; + } + [orderedCodecs addObjectsFromArray:codecs]; + + return [orderedCodecs copy]; +} + +@end diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoDecoderH265.h b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoDecoderH265.h new file mode 100644 index 0000000000..63bb07db49 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoDecoderH265.h @@ -0,0 +1,19 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" +#import "RTCVideoDecoder.h" + +RTC_OBJC_EXPORT +API_AVAILABLE(ios(11.0)) +@interface TGRTCVideoDecoderH265 : NSObject +@end diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoDecoderH265.mm b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoDecoderH265.mm new file mode 100644 index 0000000000..5de95d65eb --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoDecoderH265.mm @@ -0,0 +1,326 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "TGRTCVideoDecoderH265.h" + +#import + +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#import "helpers.h" +#import "helpers/scoped_cftyperef.h" + +#if defined(WEBRTC_IOS) +#import "helpers/UIDevice+RTCDevice.h" +#endif + +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "sdk/objc/components/video_codec/nalu_rewriter.h" + +typedef void (^TGRTCVideoDecoderRequestKeyframeCallback)(); + +// Struct that we pass to the decoder per frame to decode. We receive it again +// in the decoder callback. +struct RTCH265FrameDecodeParams { + RTCH265FrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts, TGRTCVideoDecoderRequestKeyframeCallback requestFrame) + : callback(cb), timestamp(ts), requestFrame(requestFrame) {} + RTCVideoDecoderCallback callback; + int64_t timestamp; + TGRTCVideoDecoderRequestKeyframeCallback requestFrame; +}; + +// This is the callback function that VideoToolbox calls when decode is +// complete. +static void tg_h265DecompressionOutputCallback(void* decoder, + void* params, + OSStatus status, + VTDecodeInfoFlags infoFlags, + CVImageBufferRef imageBuffer, + CMTime timestamp, + CMTime duration) { + std::unique_ptr decodeParams( + reinterpret_cast(params)); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; + if (status == -12909) { + decodeParams->requestFrame(); + } + return; + } + // TODO(tkchin): Handle CVO properly. + RTCCVPixelBuffer* frameBuffer = + [[RTCCVPixelBuffer alloc] initWithPixelBuffer:imageBuffer]; + RTCVideoFrame* decodedFrame = [[RTCVideoFrame alloc] + initWithBuffer:frameBuffer + rotation:RTCVideoRotation_0 + timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec]; + decodedFrame.timeStamp = decodeParams->timestamp; + decodeParams->callback(decodedFrame); +} + +@interface TGRTCVideoDecoderH265RequestKeyframeHolder : NSObject + +@property (nonatomic) NSLock *lock; +@property (nonatomic) bool shouldRequestKeyframe; + +@end + +@implementation TGRTCVideoDecoderH265RequestKeyframeHolder + +- (instancetype)init { + self = [super init]; + if (self != nil) { + _lock = [[NSLock alloc] init]; + } + return self; +} + +@end + +// Decoder. +@implementation TGRTCVideoDecoderH265 { + CMVideoFormatDescriptionRef _videoFormat; + VTDecompressionSessionRef _decompressionSession; + RTCVideoDecoderCallback _callback; + TGRTCVideoDecoderH265RequestKeyframeHolder *_requestKeyframeHolder; + TGRTCVideoDecoderRequestKeyframeCallback _requestFrame; + OSStatus _error; +} + +- (instancetype)init { + if (self = [super init]) { + _requestKeyframeHolder = [[TGRTCVideoDecoderH265RequestKeyframeHolder alloc] init]; + TGRTCVideoDecoderH265RequestKeyframeHolder *requestKeyframeHolder = _requestKeyframeHolder; + _requestFrame = ^{ + [requestKeyframeHolder.lock lock]; + requestKeyframeHolder.shouldRequestKeyframe = true; + [requestKeyframeHolder.lock unlock]; + }; + } + + return self; +} + +- (void)dealloc { + [self destroyDecompressionSession]; + [self setVideoFormat:nullptr]; +} + +- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores { + return WEBRTC_VIDEO_CODEC_OK; +} + +- (NSInteger)decode:(RTCEncodedImage*)inputImage + missingFrames:(BOOL)missingFrames + codecSpecificInfo:(__nullable id)info + renderTimeMs:(int64_t)renderTimeMs { + RTC_DCHECK(inputImage.buffer); + + if (_error != noErr) { + RTC_LOG(LS_WARNING) << "Last frame decode failed."; + _error = noErr; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + rtc::ScopedCFTypeRef inputFormat = + rtc::ScopedCF(webrtc::CreateH265VideoFormatDescription( + (uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length)); + if (inputFormat) { + CMVideoDimensions dimensions = + CMVideoFormatDescriptionGetDimensions(inputFormat.get()); + RTC_LOG(LS_INFO) << "Resolution: " << dimensions.width << " x " + << dimensions.height; + // Check if the video format has changed, and reinitialize decoder if + // needed. + if (!CMFormatDescriptionEqual(inputFormat.get(), _videoFormat)) { + [self setVideoFormat:inputFormat.get()]; + int resetDecompressionSessionError = [self resetDecompressionSession]; + if (resetDecompressionSessionError != WEBRTC_VIDEO_CODEC_OK) { + return resetDecompressionSessionError; + } + } + } + if (!_videoFormat) { + // We received a frame but we don't have format information so we can't + // decode it. + // This can happen after backgrounding. We need to wait for the next + // sps/pps before we can resume so we request a keyframe by returning an + // error. + RTC_LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + CMSampleBufferRef sampleBuffer = nullptr; + if (!webrtc::H265AnnexBBufferToCMSampleBuffer( + (uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length, + _videoFormat, &sampleBuffer)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(sampleBuffer); + VTDecodeFrameFlags decodeFlags = + kVTDecodeFrame_EnableAsynchronousDecompression; + std::unique_ptr frameDecodeParams; + frameDecodeParams.reset( + new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp, _requestFrame)); + OSStatus status = VTDecompressionSessionDecodeFrame( + _decompressionSession, sampleBuffer, decodeFlags, + frameDecodeParams.release(), nullptr); +#if defined(WEBRTC_IOS) + // Re-initialize the decoder if we have an invalid session while the app is + // active and retry the decode request. + if (status == kVTInvalidSessionErr && + [self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) { + frameDecodeParams.reset( + new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp, _requestFrame)); + status = VTDecompressionSessionDecodeFrame( + _decompressionSession, sampleBuffer, decodeFlags, + frameDecodeParams.release(), nullptr); + } +#endif + CFRelease(sampleBuffer); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to decode frame with code: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + bool requestKeyframe = false; + + [_requestKeyframeHolder.lock lock]; + if (_requestKeyframeHolder.shouldRequestKeyframe) { + _requestKeyframeHolder.shouldRequestKeyframe = false; + requestKeyframe = true; + } + [_requestKeyframeHolder.lock unlock]; + + if (requestKeyframe) { + RTC_LOG(LS_ERROR) << "Decoder asynchronously asked to request keyframe"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)setCallback:(RTCVideoDecoderCallback)callback { + _callback = callback; +} + +- (NSInteger)releaseDecoder { + // Need to invalidate the session so that callbacks no longer occur and it + // is safe to null out the callback. + [self destroyDecompressionSession]; + [self setVideoFormat:nullptr]; + _callback = nullptr; + return WEBRTC_VIDEO_CODEC_OK; +} + +#pragma mark - Private + +- (int)resetDecompressionSession { + [self destroyDecompressionSession]; + + // Need to wait for the first SPS to initialize decoder. + if (!_videoFormat) { + return WEBRTC_VIDEO_CODEC_OK; + } + + // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder + // create pixel buffers with GPU backed memory. The intent here is to pass + // the pixel buffers directly so we avoid a texture upload later during + // rendering. This currently is moot because we are converting back to an + // I420 frame after decode, but eventually we will be able to plumb + // CVPixelBuffers directly to the renderer. + // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that + // we can pass CVPixelBuffers as native handles in decoder output. + static size_t const attributesSize = 3; + CFTypeRef keys[attributesSize] = { +#if defined(WEBRTC_IOS) + kCVPixelBufferOpenGLESCompatibilityKey, +#elif defined(WEBRTC_MAC) + kCVPixelBufferOpenGLCompatibilityKey, +#endif + kCVPixelBufferIOSurfacePropertiesKey, + kCVPixelBufferPixelFormatTypeKey + }; + CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); + int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + CFNumberRef pixelFormat = + CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); + CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, + pixelFormat}; + CFDictionaryRef attributes = + CreateCFTypeDictionary(keys, values, attributesSize); + if (ioSurfaceValue) { + CFRelease(ioSurfaceValue); + ioSurfaceValue = nullptr; + } + if (pixelFormat) { + CFRelease(pixelFormat); + pixelFormat = nullptr; + } + VTDecompressionOutputCallbackRecord record = { + tg_h265DecompressionOutputCallback, + nullptr, + }; + OSStatus status = + VTDecompressionSessionCreate(nullptr, _videoFormat, nullptr, attributes, + &record, &_decompressionSession); + CFRelease(attributes); + if (status != noErr) { + [self destroyDecompressionSession]; + return WEBRTC_VIDEO_CODEC_ERROR; + } + [self configureDecompressionSession]; + + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)configureDecompressionSession { + RTC_DCHECK(_decompressionSession); +#if defined(WEBRTC_IOS) + // VTSessionSetProperty(_decompressionSession, + // kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); +#endif +} + +- (void)destroyDecompressionSession { + if (_decompressionSession) { +#if defined(WEBRTC_IOS) + if ([UIDevice isIOS11OrLater]) { + VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession); + } +#endif + VTDecompressionSessionInvalidate(_decompressionSession); + CFRelease(_decompressionSession); + _decompressionSession = nullptr; + } +} + +- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat { + if (_videoFormat == videoFormat) { + return; + } + if (_videoFormat) { + CFRelease(_videoFormat); + } + _videoFormat = videoFormat; + if (_videoFormat) { + CFRetain(_videoFormat); + } +} + +- (NSString*)implementationName { + return @"VideoToolbox"; +} + +@end diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoEncoderH265.h b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoEncoderH265.h new file mode 100644 index 0000000000..b68192298a --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoEncoderH265.h @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import + +#import "RTCMacros.h" +#import "RTCVideoCodecInfo.h" +#import "RTCVideoEncoder.h" + +RTC_OBJC_EXPORT +API_AVAILABLE(ios(11.0)) +@interface TGRTCVideoEncoderH265 : NSObject + +- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo; + +@end diff --git a/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoEncoderH265.mm b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoEncoderH265.mm new file mode 100644 index 0000000000..d9ef797336 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Apple/TGRTCVideoEncoderH265.mm @@ -0,0 +1,613 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "TGRTCVideoEncoderH265.h" + +#import +#include + +#import "RTCCodecSpecificInfoH265.h" +#import "api/peerconnection/RTCRtpFragmentationHeader+Private.h" +#import "api/peerconnection/RTCVideoCodecInfo+Private.h" +#import "base/RTCI420Buffer.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/video_frame_buffer/RTCCVPixelBuffer.h" +#import "helpers.h" +#if defined(WEBRTC_IOS) +#import "helpers/UIDevice+RTCDevice.h" +#endif + +#include "common_video/h264/profile_level_id.h" +#include "common_video/h265/h265_bitstream_parser.h" +#include "common_video/include/bitrate_adjuster.h" +#include "libyuv/convert_from.h" +#include "modules/include/module_common_types.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/buffer.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" +#include "system_wrappers/include/clock.h" + +@interface TGRTCVideoEncoderH265 () + +- (void)frameWasEncoded:(OSStatus)status + flags:(VTEncodeInfoFlags)infoFlags + sampleBuffer:(CMSampleBufferRef)sampleBuffer + width:(int32_t)width + height:(int32_t)height + renderTimeMs:(int64_t)renderTimeMs + timestamp:(uint32_t)timestamp + rotation:(RTCVideoRotation)rotation; + +@end + +namespace { // anonymous namespace + +// The ratio between kVTCompressionPropertyKey_DataRateLimits and +// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher +// than the average bit rate to avoid undershooting the target. +const float kLimitToAverageBitRateFactor = 1.5f; +// These thresholds deviate from the default h265 QP thresholds, as they +// have been found to work better on devices that support VideoToolbox +const int kLowh265QpThreshold = 28; +const int kHighh265QpThreshold = 39; + +// Struct that we pass to the encoder per frame to encode. We receive it again +// in the encoder callback. +struct API_AVAILABLE(ios(11.0)) RTCFrameEncodeParams { + RTCFrameEncodeParams(TGRTCVideoEncoderH265* e, + int32_t w, + int32_t h, + int64_t rtms, + uint32_t ts, + RTCVideoRotation r) + : encoder(e), + width(w), + height(h), + render_time_ms(rtms), + timestamp(ts), + rotation(r) {} + + TGRTCVideoEncoderH265* encoder; + int32_t width; + int32_t height; + int64_t render_time_ms; + uint32_t timestamp; + RTCVideoRotation rotation; +}; + +// We receive I420Frames as input, but we need to feed CVPixelBuffers into the +// encoder. This performs the copy and format conversion. +// TODO(tkchin): See if encoder will accept i420 frames and compare performance. +bool CopyVideoFrameToPixelBuffer(id frameBuffer, + CVPixelBufferRef pixelBuffer) { + RTC_DCHECK(pixelBuffer); + RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); + RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), + frameBuffer.height); + RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), + frameBuffer.width); + + CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + return false; + } + + uint8_t* dstY = reinterpret_cast( + CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); + int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); + uint8_t* dstUV = reinterpret_cast( + CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); + int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); + // Convert I420 to NV12. + int ret = libyuv::I420ToNV12( + frameBuffer.dataY, frameBuffer.strideY, frameBuffer.dataU, + frameBuffer.strideU, frameBuffer.dataV, frameBuffer.strideV, dstY, + dstStrideY, dstUV, dstStrideUV, frameBuffer.width, frameBuffer.height); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + if (ret) { + RTC_LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; + return false; + } + return true; +} + +CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { + if (!pixel_buffer_pool) { + RTC_LOG(LS_ERROR) << "Failed to get pixel buffer pool."; + return nullptr; + } + CVPixelBufferRef pixel_buffer; + CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, + &pixel_buffer); + if (ret != kCVReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; + // We probably want to drop frames here, since failure probably means + // that the pool is empty. + return nullptr; + } + return pixel_buffer; +} + +// This is the callback function that VideoToolbox calls when encode is +// complete. From inspection this happens on its own queue. +void compressionOutputCallback(void* encoder, + void* params, + OSStatus status, + VTEncodeInfoFlags infoFlags, + CMSampleBufferRef sampleBuffer) + API_AVAILABLE(ios(11.0)) { + RTC_CHECK(params); + std::unique_ptr encodeParams( + reinterpret_cast(params)); + RTC_CHECK(encodeParams->encoder); + [encodeParams->encoder frameWasEncoded:status + flags:infoFlags + sampleBuffer:sampleBuffer + width:encodeParams->width + height:encodeParams->height + renderTimeMs:encodeParams->render_time_ms + timestamp:encodeParams->timestamp + rotation:encodeParams->rotation]; +} +} // namespace + +@implementation TGRTCVideoEncoderH265 { + RTCVideoCodecInfo* _codecInfo; + std::unique_ptr _bitrateAdjuster; + uint32_t _targetBitrateBps; + uint32_t _encoderBitrateBps; + CFStringRef _profile; + RTCVideoEncoderCallback _callback; + int32_t _width; + int32_t _height; + VTCompressionSessionRef _compressionSession; + RTCVideoCodecMode _mode; + int framesLeft; + + webrtc::H265BitstreamParser _h265BitstreamParser; + std::vector _nv12ScaleBuffer; +} + +// .5 is set as a mininum to prevent overcompensating for large temporary +// overshoots. We don't want to degrade video quality too badly. +// .95 is set to prevent oscillations. When a lower bitrate is set on the +// encoder than previously set, its output seems to have a brief period of +// drastically reduced bitrate, so we want to avoid that. In steady state +// conditions, 0.95 seems to give us better overall bitrate over long periods +// of time. +- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo*)codecInfo { + if (self = [super init]) { + _codecInfo = codecInfo; + _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95)); + RTC_CHECK([codecInfo.name isEqualToString:@"H265"]); + } + return self; +} + +- (void)dealloc { + [self destroyCompressionSession]; +} + +- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings*)settings + numberOfCores:(int)numberOfCores { + RTC_DCHECK(settings); + RTC_DCHECK([settings.name isEqualToString:@"H265"]); + + _width = settings.width; + _height = settings.height; + _mode = settings.mode; + + // We can only set average bitrate on the HW encoder. + _targetBitrateBps = settings.startBitrate; + _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); + + // TODO(tkchin): Try setting payload size via + // kVTCompressionPropertyKey_Maxh265SliceBytes. + + return [self resetCompressionSession]; +} + +- (NSInteger)encode:(RTCVideoFrame*)frame + codecSpecificInfo:(id)codecSpecificInfo + frameTypes:(NSArray*)frameTypes { + RTC_DCHECK_EQ(frame.width, _width); + RTC_DCHECK_EQ(frame.height, _height); + if (!_callback || !_compressionSession) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + BOOL isKeyframeRequired = NO; + + // Get a pixel buffer from the pool and copy frame data over. + CVPixelBufferPoolRef pixelBufferPool = + VTCompressionSessionGetPixelBufferPool(_compressionSession); + +#if defined(WEBRTC_IOS) + if (!pixelBufferPool) { + // Kind of a hack. On backgrounding, the compression session seems to get + // invalidated, which causes this pool call to fail when the application + // is foregrounded and frames are being sent for encoding again. + // Resetting the session when this happens fixes the issue. + // In addition we request a keyframe so video can recover quickly. + [self resetCompressionSession]; + pixelBufferPool = + VTCompressionSessionGetPixelBufferPool(_compressionSession); + isKeyframeRequired = YES; + RTC_LOG(LS_INFO) << "Resetting compression session due to invalid pool."; + } +#endif + + CVPixelBufferRef pixelBuffer = nullptr; + if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + // Native frame buffer + RTCCVPixelBuffer* rtcPixelBuffer = (RTCCVPixelBuffer*)frame.buffer; + if (![rtcPixelBuffer requiresCropping]) { + // This pixel buffer might have a higher resolution than what the + // compression session is configured to. The compression session can + // handle that and will output encoded frames in the configured + // resolution regardless of the input pixel buffer resolution. + pixelBuffer = rtcPixelBuffer.pixelBuffer; + CVBufferRetain(pixelBuffer); + } else { + // Cropping required, we need to crop and scale to a new pixel buffer. + pixelBuffer = CreatePixelBuffer(pixelBufferPool); + if (!pixelBuffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + int dstWidth = CVPixelBufferGetWidth(pixelBuffer); + int dstHeight = CVPixelBufferGetHeight(pixelBuffer); + if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) { + int size = + [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth + height:dstHeight]; + _nv12ScaleBuffer.resize(size); + } else { + _nv12ScaleBuffer.clear(); + } + _nv12ScaleBuffer.shrink_to_fit(); + if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer + withTempBuffer:_nv12ScaleBuffer.data()]) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + } + + if (!pixelBuffer) { + // We did not have a native frame buffer + pixelBuffer = CreatePixelBuffer(pixelBufferPool); + if (!pixelBuffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(pixelBuffer); + if (!CopyVideoFrameToPixelBuffer([frame.buffer toI420], pixelBuffer)) { + RTC_LOG(LS_ERROR) << "Failed to copy frame data."; + CVBufferRelease(pixelBuffer); + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + // Check if we need a keyframe. + if (!isKeyframeRequired && frameTypes) { + for (NSNumber* frameType in frameTypes) { + if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) { + isKeyframeRequired = YES; + break; + } + } + } + + CMTime presentationTimeStamp = + CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000); + CFDictionaryRef frameProperties = nullptr; + if (isKeyframeRequired) { + CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; + CFTypeRef values[] = {kCFBooleanTrue}; + frameProperties = CreateCFTypeDictionary(keys, values, 1); + } + + std::unique_ptr encodeParams; + encodeParams.reset(new RTCFrameEncodeParams( + self, _width, _height, frame.timeStampNs / rtc::kNumNanosecsPerMillisec, + frame.timeStamp, frame.rotation)); + + // Update the bitrate if needed. + [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; + + OSStatus status = VTCompressionSessionEncodeFrame( + _compressionSession, pixelBuffer, presentationTimeStamp, kCMTimeInvalid, + frameProperties, encodeParams.release(), nullptr); + if (frameProperties) { + CFRelease(frameProperties); + } + if (pixelBuffer) { + CVBufferRelease(pixelBuffer); + } + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to encode frame with code: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)setCallback:(RTCVideoEncoderCallback)callback { + _callback = callback; +} + +- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { + _targetBitrateBps = 1000 * bitrateKbit; + _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); + [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; + return WEBRTC_VIDEO_CODEC_OK; +} + +#pragma mark - Private + +- (NSInteger)releaseEncoder { + // Need to destroy so that the session is invalidated and won't use the + // callback anymore. Do not remove callback until the session is invalidated + // since async encoder callbacks can occur until invalidation. + [self destroyCompressionSession]; + _callback = nullptr; + return WEBRTC_VIDEO_CODEC_OK; +} + +- (int)resetCompressionSession { + [self destroyCompressionSession]; + + // Set source image buffer attributes. These attributes will be present on + // buffers retrieved from the encoder's pixel buffer pool. + const size_t attributesSize = 3; + CFTypeRef keys[attributesSize] = { +#if defined(WEBRTC_IOS) + kCVPixelBufferOpenGLESCompatibilityKey, +#elif defined(WEBRTC_MAC) + kCVPixelBufferOpenGLCompatibilityKey, +#endif + kCVPixelBufferIOSurfacePropertiesKey, + kCVPixelBufferPixelFormatTypeKey + }; + CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); + int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + CFNumberRef pixelFormat = + CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); + CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, + pixelFormat}; + CFDictionaryRef sourceAttributes = + CreateCFTypeDictionary(keys, values, attributesSize); + if (ioSurfaceValue) { + CFRelease(ioSurfaceValue); + ioSurfaceValue = nullptr; + } + if (pixelFormat) { + CFRelease(pixelFormat); + pixelFormat = nullptr; + } + CFMutableDictionaryRef encoder_specs = nullptr; +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + // Currently hw accl is supported above 360p on mac, below 360p + // the compression session will be created with hw accl disabled. + encoder_specs = + CFDictionaryCreateMutable(nullptr, 1, &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + CFDictionarySetValue( + encoder_specs, + kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, + kCFBooleanTrue); +#endif + OSStatus status = VTCompressionSessionCreate( + nullptr, // use default allocator + _width, _height, kCMVideoCodecType_HEVC, + encoder_specs, // use hardware accelerated encoder if available + sourceAttributes, + nullptr, // use default compressed data allocator + compressionOutputCallback, nullptr, &_compressionSession); + if (sourceAttributes) { + CFRelease(sourceAttributes); + sourceAttributes = nullptr; + } + if (encoder_specs) { + CFRelease(encoder_specs); + encoder_specs = nullptr; + } + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + CFBooleanRef hwaccl_enabled = nullptr; + status = VTSessionCopyProperty( + _compressionSession, + kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, nullptr, + &hwaccl_enabled); + if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) { + RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled"; + } else { + RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled"; + } +#endif + [self configureCompressionSession]; + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)configureCompressionSession { + RTC_DCHECK(_compressionSession); + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, + true); + // SetVTSessionProperty(_compressionSession, + // kVTCompressionPropertyKey_ProfileLevel, _profile); + SetVTSessionProperty(_compressionSession, + kVTCompressionPropertyKey_AllowFrameReordering, false); + [self setEncoderBitrateBps:_targetBitrateBps]; + // TODO(tkchin): Look at entropy mode and colorspace matrices. + // TODO(tkchin): Investigate to see if there's any way to make this work. + // May need it to interop with Android. Currently this call just fails. + // On inspecting encoder output on iOS8, this value is set to 6. + // internal::SetVTSessionProperty(compression_session_, + // kVTCompressionPropertyKey_MaxFrameDelayCount, + // 1); + + // Set a relatively large value for keyframe emission (7200 frames or 4 + // minutes). + SetVTSessionProperty(_compressionSession, + kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); + SetVTSessionProperty(_compressionSession, + kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, + 240); + OSStatus status = + VTCompressionSessionPrepareToEncodeFrames(_compressionSession); + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Compression session failed to prepare encode frames."; + } +} + +- (void)destroyCompressionSession { + if (_compressionSession) { + VTCompressionSessionInvalidate(_compressionSession); + CFRelease(_compressionSession); + _compressionSession = nullptr; + } +} + +- (NSString*)implementationName { + return @"VideoToolbox"; +} + +- (void)setBitrateBps:(uint32_t)bitrateBps { + if (_encoderBitrateBps != bitrateBps) { + [self setEncoderBitrateBps:bitrateBps]; + } +} + +- (void)setEncoderBitrateBps:(uint32_t)bitrateBps { + if (_compressionSession) { + SetVTSessionProperty(_compressionSession, + kVTCompressionPropertyKey_AverageBitRate, bitrateBps); + + // TODO(tkchin): Add a helper method to set array value. + int64_t dataLimitBytesPerSecondValue = + static_cast(bitrateBps * kLimitToAverageBitRateFactor / 8); + CFNumberRef bytesPerSecond = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, + &dataLimitBytesPerSecondValue); + int64_t oneSecondValue = 1; + CFNumberRef oneSecond = CFNumberCreate( + kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue); + const void* nums[2] = {bytesPerSecond, oneSecond}; + CFArrayRef dataRateLimits = + CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); + OSStatus status = VTSessionSetProperty( + _compressionSession, kVTCompressionPropertyKey_DataRateLimits, + dataRateLimits); + if (bytesPerSecond) { + CFRelease(bytesPerSecond); + } + if (oneSecond) { + CFRelease(oneSecond); + } + if (dataRateLimits) { + CFRelease(dataRateLimits); + } + if (status != noErr) { + RTC_LOG(LS_ERROR) << "Failed to set data rate limit"; + } + + _encoderBitrateBps = bitrateBps; + } +} + +- (void)frameWasEncoded:(OSStatus)status + flags:(VTEncodeInfoFlags)infoFlags + sampleBuffer:(CMSampleBufferRef)sampleBuffer + width:(int32_t)width + height:(int32_t)height + renderTimeMs:(int64_t)renderTimeMs + timestamp:(uint32_t)timestamp + rotation:(RTCVideoRotation)rotation { + if (status != noErr) { + RTC_LOG(LS_ERROR) << "h265 encode failed."; + return; + } + if (infoFlags & kVTEncodeInfo_FrameDropped) { + RTC_LOG(LS_INFO) << "h265 encoder dropped a frame."; + return; + } + + BOOL isKeyframe = NO; + CFArrayRef attachments = + CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); + if (attachments != nullptr && CFArrayGetCount(attachments)) { + CFDictionaryRef attachment = + static_cast(CFArrayGetValueAtIndex(attachments, 0)); + isKeyframe = + !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); + } + + if (isKeyframe) { + RTC_LOG(LS_INFO) << "Generated keyframe"; + } + + // Convert the sample buffer into a buffer suitable for RTP packetization. + // TODO(tkchin): Allocate buffers through a pool. + std::unique_ptr buffer(new rtc::Buffer()); + RTCRtpFragmentationHeader* header; + { + std::unique_ptr header_cpp; + bool result = H265CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, + buffer.get(), &header_cpp); + header = [[RTCRtpFragmentationHeader alloc] + initWithNativeFragmentationHeader:header_cpp.get()]; + if (!result) { + RTC_LOG(LS_ERROR) << "Failed to convert sample buffer."; + return; + } + } + + RTCEncodedImage* frame = [[RTCEncodedImage alloc] init]; + frame.buffer = [NSData dataWithBytesNoCopy:buffer->data() + length:buffer->size() + freeWhenDone:NO]; + frame.encodedWidth = width; + frame.encodedHeight = height; + frame.completeFrame = YES; + frame.frameType = + isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta; + frame.captureTimeMs = renderTimeMs; + frame.timeStamp = timestamp; + frame.rotation = rotation; + frame.contentType = (_mode == RTCVideoCodecModeScreensharing) + ? RTCVideoContentTypeScreenshare + : RTCVideoContentTypeUnspecified; + frame.flags = webrtc::VideoSendTiming::kInvalid; + + int qp; + _h265BitstreamParser.ParseBitstream(buffer->data(), buffer->size()); + _h265BitstreamParser.GetLastSliceQp(&qp); + frame.qp = @(qp); + + BOOL res = _callback(frame, [[RTCCodecSpecificInfoH265 alloc] init], header); + if (!res) { + RTC_LOG(LS_ERROR) << "Encode callback failed."; + return; + } + _bitrateAdjuster->Update(frame.buffer.length); +} + +- (RTCVideoEncoderQpThresholds*)scalingSettings { + return [[RTCVideoEncoderQpThresholds alloc] + initWithThresholdsLow:kLowh265QpThreshold + high:kHighh265QpThreshold]; +} + +@end diff --git a/submodules/TgVoipWebrtc/Impl/CodecsApple.mm b/submodules/TgVoipWebrtc/Impl/CodecsApple.mm index e4f4cc136d..519c836f44 100644 --- a/submodules/TgVoipWebrtc/Impl/CodecsApple.mm +++ b/submodules/TgVoipWebrtc/Impl/CodecsApple.mm @@ -16,10 +16,8 @@ #include "api/video/builtin_video_bitrate_allocator_factory.h" #include "api/video/video_bitrate_allocation.h" -#include "sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.h" -#include "sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.h" -#include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" -#include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" +#include "Apple/TGRTCDefaultVideoEncoderFactory.h" +#include "Apple/TGRTCDefaultVideoDecoderFactory.h" #include "sdk/objc/native/api/video_encoder_factory.h" #include "sdk/objc/native/api/video_decoder_factory.h" @@ -156,11 +154,11 @@ void configurePlatformAudio() { } std::unique_ptr makeVideoEncoderFactory() { - return webrtc::ObjCToNativeVideoEncoderFactory([[RTCDefaultVideoEncoderFactory alloc] init]); + return webrtc::ObjCToNativeVideoEncoderFactory([[TGRTCDefaultVideoEncoderFactory alloc] init]); } std::unique_ptr makeVideoDecoderFactory() { - return webrtc::ObjCToNativeVideoDecoderFactory([[RTCDefaultVideoDecoderFactory alloc] init]); + return webrtc::ObjCToNativeVideoDecoderFactory([[TGRTCDefaultVideoDecoderFactory alloc] init]); } rtc::scoped_refptr makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread) {