From 6d1732d8303da6d48ef23580171e5836d9581e1f Mon Sep 17 00:00:00 2001 From: Isaac <> Date: Thu, 13 Mar 2025 00:34:46 +0100 Subject: [PATCH 1/5] Fix default to speaker (cherry picked from commit 950c3d9f1e20eeea79feeda7fbe1dfc3f1187b6d) --- .../Sources/PresentationCall.swift | 38 ++++++++++++++++++- .../Sources/PresentationGroupCall.swift | 2 +- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/submodules/TelegramCallsUI/Sources/PresentationCall.swift b/submodules/TelegramCallsUI/Sources/PresentationCall.swift index 382f61463e..de7be12e55 100644 --- a/submodules/TelegramCallsUI/Sources/PresentationCall.swift +++ b/submodules/TelegramCallsUI/Sources/PresentationCall.swift @@ -19,6 +19,8 @@ public final class SharedCallAudioContext { let audioDevice: OngoingCallContext.AudioDevice? let callKitIntegration: CallKitIntegration? + private let defaultToSpeaker: Bool + private var audioSessionDisposable: Disposable? private var audioSessionShouldBeActiveDisposable: Disposable? private var isAudioSessionActiveDisposable: Disposable? @@ -40,10 +42,17 @@ public final class SharedCallAudioContext { } private let audioSessionShouldBeActive = Promise(true) + private var initialSetupTimer: Foundation.Timer? - init(audioSession: ManagedAudioSession, callKitIntegration: CallKitIntegration?) { + init(audioSession: ManagedAudioSession, callKitIntegration: CallKitIntegration?, defaultToSpeaker: Bool = false) { self.callKitIntegration = callKitIntegration self.audioDevice = OngoingCallContext.AudioDevice.create(enableSystemMute: false) + self.defaultToSpeaker = defaultToSpeaker + + if defaultToSpeaker { + self.didSetCurrentAudioOutputValue = true + self.currentAudioOutputValue = .speaker + } var didReceiveAudioOutputs = false self.audioSessionDisposable = audioSession.push(audioSessionType: .voiceCall, manualActivate: { [weak self] control in @@ -72,6 +81,27 @@ public final class SharedCallAudioContext { audioSessionActive = .single(true) } self.isAudioSessionActivePromise.set(audioSessionActive) + + self.initialSetupTimer?.invalidate() + self.initialSetupTimer = Foundation.Timer(timeInterval: 0.5, repeats: false, block: { [weak self] _ in + guard let self else { + return + } + + if self.defaultToSpeaker, let audioSessionControl = self.audioSessionControl { + self.currentAudioOutputValue = .speaker + self.didSetCurrentAudioOutputValue = true + + if let callKitIntegration = self.callKitIntegration { + if self.didSetCurrentAudioOutputValue { + callKitIntegration.applyVoiceChatOutputMode(outputMode: .custom(self.currentAudioOutputValue)) + } + } else { + audioSessionControl.setOutputMode(.custom(self.currentAudioOutputValue)) + audioSessionControl.setup(synchronous: true) + } + } + }) } } }, deactivate: { [weak self] _ in @@ -160,9 +190,13 @@ public final class SharedCallAudioContext { self.audioSessionShouldBeActiveDisposable?.dispose() self.isAudioSessionActiveDisposable?.dispose() self.audioOutputStateDisposable?.dispose() + self.initialSetupTimer?.invalidate() } func setCurrentAudioOutput(_ output: AudioSessionOutput) { + self.initialSetupTimer?.invalidate() + self.initialSetupTimer = nil + guard self.currentAudioOutputValue != output else { return } @@ -427,7 +461,7 @@ public final class PresentationCallImpl: PresentationCall { if let data = context.currentAppConfiguration.with({ $0 }).data, let _ = data["ios_killswitch_disable_call_device"] { self.sharedAudioContext = nil } else { - self.sharedAudioContext = SharedCallAudioContext(audioSession: audioSession, callKitIntegration: callKitIntegration) + self.sharedAudioContext = SharedCallAudioContext(audioSession: audioSession, callKitIntegration: callKitIntegration, defaultToSpeaker: startWithVideo || initialState?.type == .video) } if let _ = self.sharedAudioContext { diff --git a/submodules/TelegramCallsUI/Sources/PresentationGroupCall.swift b/submodules/TelegramCallsUI/Sources/PresentationGroupCall.swift index 8893bb0cf0..7374021ee5 100644 --- a/submodules/TelegramCallsUI/Sources/PresentationGroupCall.swift +++ b/submodules/TelegramCallsUI/Sources/PresentationGroupCall.swift @@ -1134,7 +1134,7 @@ public final class PresentationGroupCallImpl: PresentationGroupCall { } if useSharedAudio { - let sharedAudioContextValue = SharedCallAudioContext(audioSession: audioSession, callKitIntegration: callKitIntegration) + let sharedAudioContextValue = SharedCallAudioContext(audioSession: audioSession, callKitIntegration: callKitIntegration, defaultToSpeaker: true) sharedAudioContext = sharedAudioContextValue } } From daaa6a7e2e42ea9e762a7cf7de62a2afa3d1466f Mon Sep 17 00:00:00 2001 From: Isaac <> Date: Fri, 14 Mar 2025 15:34:16 +0100 Subject: [PATCH 2/5] IOSurface experiment --- .../Sources/DebugController.swift | 21 +- .../Sources/ReactionContextNode.swift | 1 + .../ImageDCT/Sources/YuvConversion.m | 13 +- .../Sources/EmojiTextAttachmentView.swift | 15 +- .../Sources/EmojiKeyboardItemLayer.swift | 16 +- .../Sources/MultiAnimationRenderer.swift | 331 +++++++++++++++++- 6 files changed, 354 insertions(+), 43 deletions(-) diff --git a/submodules/DebugSettingsUI/Sources/DebugController.swift b/submodules/DebugSettingsUI/Sources/DebugController.swift index 56398e6f7c..6cf74b405b 100644 --- a/submodules/DebugSettingsUI/Sources/DebugController.swift +++ b/submodules/DebugSettingsUI/Sources/DebugController.swift @@ -98,7 +98,7 @@ private enum DebugControllerEntry: ItemListNodeEntry { case browserExperiment(Bool) case localTranscription(Bool) case enableReactionOverrides(Bool) - case storiesExperiment(Bool) + case compressedEmojiCache(Bool) case storiesJpegExperiment(Bool) case conferenceDebug(Bool) case enableQuickReactionSwitch(Bool) @@ -133,7 +133,7 @@ private enum DebugControllerEntry: ItemListNodeEntry { return DebugControllerSection.web.rawValue case .keepChatNavigationStack, .skipReadHistory, .dustEffect, .crashOnSlowQueries, .crashOnMemoryPressure: return DebugControllerSection.experiments.rawValue - case .clearTips, .resetNotifications, .crash, .fillLocalSavedMessageCache, .resetDatabase, .resetDatabaseAndCache, .resetHoles, .resetTagHoles, .reindexUnread, .resetCacheIndex, .reindexCache, .resetBiometricsData, .optimizeDatabase, .photoPreview, .knockoutWallpaper, .storiesExperiment, .storiesJpegExperiment, .conferenceDebug, .enableQuickReactionSwitch, .experimentalCompatibility, .enableDebugDataDisplay, .rippleEffect, .browserExperiment, .localTranscription, .enableReactionOverrides, .restorePurchases, .disableReloginTokens, .liveStreamV2, .experimentalCallMute, .playerV2, .devRequests, .fakeAds, .enableLocalTranslation: + case .clearTips, .resetNotifications, .crash, .fillLocalSavedMessageCache, .resetDatabase, .resetDatabaseAndCache, .resetHoles, .resetTagHoles, .reindexUnread, .resetCacheIndex, .reindexCache, .resetBiometricsData, .optimizeDatabase, .photoPreview, .knockoutWallpaper, .compressedEmojiCache, .storiesJpegExperiment, .conferenceDebug, .enableQuickReactionSwitch, .experimentalCompatibility, .enableDebugDataDisplay, .rippleEffect, .browserExperiment, .localTranscription, .enableReactionOverrides, .restorePurchases, .disableReloginTokens, .liveStreamV2, .experimentalCallMute, .playerV2, .devRequests, .fakeAds, .enableLocalTranslation: return DebugControllerSection.experiments.rawValue case .logTranslationRecognition, .resetTranslationStates: return DebugControllerSection.translation.rawValue @@ -236,7 +236,7 @@ private enum DebugControllerEntry: ItemListNodeEntry { return 44 case .resetTranslationStates: return 45 - case .storiesExperiment: + case .compressedEmojiCache: return 46 case .storiesJpegExperiment: return 47 @@ -1288,12 +1288,12 @@ private enum DebugControllerEntry: ItemListNodeEntry { }) }).start() }) - case let .storiesExperiment(value): - return ItemListSwitchItem(presentationData: presentationData, title: "Story Search Debug", value: value, sectionId: self.section, style: .blocks, updated: { value in + case let .compressedEmojiCache(value): + return ItemListSwitchItem(presentationData: presentationData, title: "Compressed Emoji Cache", value: value, sectionId: self.section, style: .blocks, updated: { value in let _ = arguments.sharedContext.accountManager.transaction ({ transaction in transaction.updateSharedData(ApplicationSpecificSharedDataKeys.experimentalUISettings, { settings in var settings = settings?.get(ExperimentalUISettings.self) ?? ExperimentalUISettings.defaultSettings - settings.storiesExperiment = value + settings.compressedEmojiCache = value return PreferencesEntry(settings) }) }).start() @@ -1535,11 +1535,10 @@ private func debugControllerEntries(sharedContext: SharedAccountContext, present entries.append(.logTranslationRecognition(experimentalSettings.logLanguageRecognition)) entries.append(.resetTranslationStates) - if case .internal = sharedContext.applicationBindings.appBuildType { - entries.append(.storiesExperiment(experimentalSettings.storiesExperiment)) - entries.append(.storiesJpegExperiment(experimentalSettings.storiesJpegExperiment)) - entries.append(.disableReloginTokens(experimentalSettings.disableReloginTokens)) - } + entries.append(.compressedEmojiCache(experimentalSettings.compressedEmojiCache)) + entries.append(.storiesJpegExperiment(experimentalSettings.storiesJpegExperiment)) + entries.append(.disableReloginTokens(experimentalSettings.disableReloginTokens)) + entries.append(.conferenceDebug(experimentalSettings.conferenceDebug)) entries.append(.enableQuickReactionSwitch(!experimentalSettings.disableQuickReaction)) entries.append(.liveStreamV2(experimentalSettings.liveStreamV2)) diff --git a/submodules/ReactionSelectionNode/Sources/ReactionContextNode.swift b/submodules/ReactionSelectionNode/Sources/ReactionContextNode.swift index bc4608d87b..1eb61c810b 100644 --- a/submodules/ReactionSelectionNode/Sources/ReactionContextNode.swift +++ b/submodules/ReactionSelectionNode/Sources/ReactionContextNode.swift @@ -499,6 +499,7 @@ public final class ReactionContextNode: ASDisplayNode, ASScrollViewDelegate { self.animationCache = animationCache self.animationRenderer = MultiAnimationRendererImpl() + (self.animationRenderer as? MultiAnimationRendererImpl)?.useYuvA = context.sharedContext.immediateExperimentalUISettings.compressedEmojiCache self.backgroundMaskNode = ASDisplayNode() self.backgroundNode = ReactionContextBackgroundNode(largeCircleSize: largeCircleSize, smallCircleSize: smallCircleSize, maskNode: self.backgroundMaskNode) diff --git a/submodules/TelegramUI/Components/AnimationCache/ImageDCT/Sources/YuvConversion.m b/submodules/TelegramUI/Components/AnimationCache/ImageDCT/Sources/YuvConversion.m index b8558b556c..6692a6f4e5 100644 --- a/submodules/TelegramUI/Components/AnimationCache/ImageDCT/Sources/YuvConversion.m +++ b/submodules/TelegramUI/Components/AnimationCache/ImageDCT/Sources/YuvConversion.m @@ -65,8 +65,6 @@ void combineYUVAPlanesIntoARGB(uint8_t *argb, uint8_t const *inY, uint8_t const vImageConvert_YpCbCrToARGB_GenerateConversion(kvImage_YpCbCrToARGBMatrix_ITU_R_709_2, &pixelRange, &info, kvImage420Yp8_Cb8_Cr8, kvImageARGB8888, 0); }); - vImage_Error error = kvImageNoError; - vImage_Buffer destArgb; destArgb.data = (void *)argb; destArgb.width = width; @@ -97,15 +95,8 @@ void combineYUVAPlanesIntoARGB(uint8_t *argb, uint8_t const *inY, uint8_t const srcA.height = height; srcA.rowBytes = width; - error = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&srcYp, &srcCb, &srcCr, &destArgb, &info, permuteMap, 255, kvImageDoNotTile); - error = vImageOverwriteChannels_ARGB8888(&srcA, &destArgb, &destArgb, 1 << 0, kvImageDoNotTile); - - if (error != kvImageNoError) { - } - - //error = vImageOverwriteChannels_ARGB8888(&srcYp, &destArgb, &destArgb, 1 << 1, kvImageDoNotTile); - //error = vImageOverwriteChannels_ARGB8888(&srcYp, &destArgb, &destArgb, 1 << 2, kvImageDoNotTile); - //error = vImageOverwriteChannels_ARGB8888(&srcYp, &destArgb, &destArgb, 1 << 3, kvImageDoNotTile); + vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&srcYp, &srcCb, &srcCr, &destArgb, &info, permuteMap, 255, kvImageDoNotTile); + vImageOverwriteChannels_ARGB8888(&srcA, &destArgb, &destArgb, 1 << 0, kvImageDoNotTile); } void scaleImagePlane(uint8_t *outPlane, int outWidth, int outHeight, int outBytesPerRow, uint8_t const *inPlane, int inWidth, int inHeight, int inBytesPerRow) { diff --git a/submodules/TelegramUI/Components/EmojiTextAttachmentView/Sources/EmojiTextAttachmentView.swift b/submodules/TelegramUI/Components/EmojiTextAttachmentView/Sources/EmojiTextAttachmentView.swift index b942c311a5..aac8abaa7e 100644 --- a/submodules/TelegramUI/Components/EmojiTextAttachmentView/Sources/EmojiTextAttachmentView.swift +++ b/submodules/TelegramUI/Components/EmojiTextAttachmentView/Sources/EmojiTextAttachmentView.swift @@ -415,7 +415,20 @@ public final class InlineStickerItemLayer: MultiAnimationRenderTarget { } override public var contents: Any? { - didSet { + get { + return super.contents + } set(value) { + #if targetEnvironment(simulator) + if let value, CFGetTypeID(value as CFTypeRef) == CVPixelBufferGetTypeID() { + let pixelBuffer = value as! CVPixelBuffer + super.contents = CVPixelBufferGetIOSurface(pixelBuffer) + } else { + super.contents = value + } + #else + super.contents = value + #endif + if let mirrorLayer = self.mirrorLayer { mirrorLayer.contents = self.contents } diff --git a/submodules/TelegramUI/Components/EntityKeyboard/Sources/EmojiKeyboardItemLayer.swift b/submodules/TelegramUI/Components/EntityKeyboard/Sources/EmojiKeyboardItemLayer.swift index 7a584de5bf..cdf7c72a0a 100644 --- a/submodules/TelegramUI/Components/EntityKeyboard/Sources/EmojiKeyboardItemLayer.swift +++ b/submodules/TelegramUI/Components/EntityKeyboard/Sources/EmojiKeyboardItemLayer.swift @@ -10,6 +10,7 @@ import AccountContext import TelegramPresentationData import EmojiTextAttachmentView import EmojiStatusComponent +import CoreVideo final class EmojiKeyboardCloneItemLayer: SimpleLayer { } @@ -79,7 +80,20 @@ public final class EmojiKeyboardItemLayer: MultiAnimationRenderTarget { } override public var contents: Any? { - didSet { + get { + return super.contents + } set(value) { + #if targetEnvironment(simulator) + if let value, CFGetTypeID(value as CFTypeRef) == CVPixelBufferGetTypeID() { + let pixelBuffer = value as! CVPixelBuffer + super.contents = CVPixelBufferGetIOSurface(pixelBuffer) + } else { + super.contents = value + } + #else + super.contents = value + #endif + self.onContentsUpdate() if let cloneLayer = self.cloneLayer { cloneLayer.contents = self.contents diff --git a/submodules/TelegramUI/Components/MultiAnimationRenderer/Sources/MultiAnimationRenderer.swift b/submodules/TelegramUI/Components/MultiAnimationRenderer/Sources/MultiAnimationRenderer.swift index 1f39885a99..e0671fd539 100644 --- a/submodules/TelegramUI/Components/MultiAnimationRenderer/Sources/MultiAnimationRenderer.swift +++ b/submodules/TelegramUI/Components/MultiAnimationRenderer/Sources/MultiAnimationRenderer.swift @@ -4,6 +4,7 @@ import SwiftSignalKit import Display import AnimationCache import Accelerate +import IOSurface public protocol MultiAnimationRenderer: AnyObject { func add(target: MultiAnimationRenderTarget, cache: AnimationCache, itemId: String, unique: Bool, size: CGSize, fetch: @escaping (AnimationCacheFetchOptions) -> Disposable) -> Disposable @@ -89,12 +90,21 @@ private final class LoadFrameGroupTask { } } +private var yuvToRgbConversion: vImage_YpCbCrToARGB = { + var info = vImage_YpCbCrToARGB() + var pixelRange = vImage_YpCbCrPixelRange(Yp_bias: 16, CbCr_bias: 128, YpRangeMax: 235, CbCrRangeMax: 240, YpMax: 255, YpMin: 0, CbCrMax: 255, CbCrMin: 0) + vImageConvert_YpCbCrToARGB_GenerateConversion(kvImage_YpCbCrToARGBMatrix_ITU_R_709_2, &pixelRange, &info, kvImage420Yp8_Cb8_Cr8, kvImageARGB8888, 0) + return info +}() + private final class ItemAnimationContext { fileprivate final class Frame { let frame: AnimationCacheItemFrame let duration: Double - let image: UIImage - let badgeImage: UIImage? + + let contentsAsImage: UIImage? + let contentsAsCVPixelBuffer: CVPixelBuffer? + let size: CGSize var remainingDuration: Double @@ -120,11 +130,101 @@ private final class ItemAnimationContext { return nil } - self.image = image + self.contentsAsImage = image + self.contentsAsCVPixelBuffer = nil self.size = CGSize(width: CGFloat(width), height: CGFloat(height)) - self.badgeImage = nil - default: - return nil + case let .yuva(y, u, v, a): + var pixelBuffer: CVPixelBuffer? = nil + let _ = CVPixelBufferCreate(kCFAllocatorDefault, y.width, y.height, kCVPixelFormatType_420YpCbCr8VideoRange_8A_TriPlanar, [ + kCVPixelBufferIOSurfacePropertiesKey: NSDictionary() + ] as CFDictionary, &pixelBuffer) + guard let pixelBuffer else { + return nil + } + + CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) + defer { + CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0)) + } + guard let baseAddressY = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0) else { + return nil + } + guard let baseAddressCbCr = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1) else { + return nil + } + guard let baseAddressA = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 2) else { + return nil + } + + let dstBufferY = vImage_Buffer(data: UnsafeMutableRawPointer(mutating: baseAddressY), height: vImagePixelCount(y.height), width: vImagePixelCount(y.width), rowBytes: CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0)) + let dstBufferCbCr = vImage_Buffer(data: UnsafeMutableRawPointer(mutating: baseAddressCbCr), height: vImagePixelCount(y.height / 2), width: vImagePixelCount(y.width / 2), rowBytes: CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1)) + let dstBufferA = vImage_Buffer(data: UnsafeMutableRawPointer(mutating: baseAddressA), height: vImagePixelCount(y.height), width: vImagePixelCount(y.width), rowBytes: CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 2)) + + y.data.withUnsafeBytes { (yBytes: UnsafeRawBufferPointer) -> Void in + if dstBufferY.rowBytes == y.bytesPerRow { + memcpy(dstBufferY.data, yBytes.baseAddress!, yBytes.count) + } else { + for i in 0 ..< y.height { + memcpy(dstBufferY.data.advanced(by: dstBufferY.rowBytes * i), yBytes.baseAddress!.advanced(by: y.bytesPerRow * i), y.bytesPerRow) + } + } + } + + a.data.withUnsafeBytes { (aBytes: UnsafeRawBufferPointer) -> Void in + if dstBufferA.rowBytes == a.bytesPerRow { + memcpy(dstBufferA.data, aBytes.baseAddress!, aBytes.count) + } else { + for i in 0 ..< y.height { + memcpy(dstBufferA.data.advanced(by: dstBufferA.rowBytes * i), aBytes.baseAddress!.advanced(by: a.bytesPerRow * i), a.bytesPerRow) + } + } + } + + u.data.withUnsafeBytes { (uBytes: UnsafeRawBufferPointer) -> Void in + v.data.withUnsafeBytes { (vBytes: UnsafeRawBufferPointer) -> Void in + let sourceU = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: uBytes.baseAddress!), + height: vImagePixelCount(u.height), + width: vImagePixelCount(u.width), + rowBytes: u.bytesPerRow + ) + let sourceV = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: vBytes.baseAddress!), + height: vImagePixelCount(v.height), + width: vImagePixelCount(v.width), + rowBytes: v.bytesPerRow + ) + + withUnsafePointer(to: sourceU, { sourceU in + withUnsafePointer(to: sourceV, { sourceV in + var srcPlanarBuffers: [ + UnsafePointer? + ] = [sourceU, sourceV] + var destChannels: [UnsafeMutableRawPointer?] = [ + dstBufferCbCr.data.advanced(by: 1), + dstBufferCbCr.data + ] + + let channelCount = 2 + + vImageConvert_PlanarToChunky8( + &srcPlanarBuffers, + &destChannels, + UInt32(channelCount), + MemoryLayout.stride * channelCount, + vImagePixelCount(u.width), + vImagePixelCount(u.height), + dstBufferCbCr.rowBytes, + vImage_Flags(kvImageDoNotTile) + ) + }) + }) + } + } + + self.contentsAsImage = nil + self.contentsAsCVPixelBuffer = pixelBuffer + self.size = CGSize(width: CGFloat(y.width), height: CGFloat(y.height)) } } @@ -221,8 +321,163 @@ private final class ItemAnimationContext { self.blurredRepresentationValue = context.generateImage() return self.blurredRepresentationValue - default: - return nil + case let .yuva(y, u, v, a): + let blurredWidth = 12 + let blurredHeight = 12 + let size = CGSize(width: blurredWidth, height: blurredHeight) + + var sourceY = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: y.data.withUnsafeBytes { $0.baseAddress! }), + height: vImagePixelCount(y.height), + width: vImagePixelCount(y.width), + rowBytes: y.bytesPerRow + ) + + var sourceU = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: u.data.withUnsafeBytes { $0.baseAddress! }), + height: vImagePixelCount(u.height), + width: vImagePixelCount(u.width), + rowBytes: u.bytesPerRow + ) + + var sourceV = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: v.data.withUnsafeBytes { $0.baseAddress! }), + height: vImagePixelCount(v.height), + width: vImagePixelCount(v.width), + rowBytes: v.bytesPerRow + ) + + var sourceA = vImage_Buffer( + data: UnsafeMutableRawPointer(mutating: a.data.withUnsafeBytes { $0.baseAddress! }), + height: vImagePixelCount(a.height), + width: vImagePixelCount(a.width), + rowBytes: a.bytesPerRow + ) + + let scaledYData = malloc(blurredWidth * blurredHeight)! + defer { + free(scaledYData) + } + + let scaledUData = malloc(blurredWidth * blurredHeight / 4)! + defer { + free(scaledUData) + } + + let scaledVData = malloc(blurredWidth * blurredHeight / 4)! + defer { + free(scaledVData) + } + + let scaledAData = malloc(blurredWidth * blurredHeight)! + defer { + free(scaledAData) + } + + var scaledY = vImage_Buffer( + data: scaledYData, + height: vImagePixelCount(blurredHeight), + width: vImagePixelCount(blurredWidth), + rowBytes: blurredWidth + ) + + var scaledU = vImage_Buffer( + data: scaledUData, + height: vImagePixelCount(blurredHeight / 2), + width: vImagePixelCount(blurredWidth / 2), + rowBytes: blurredWidth / 2 + ) + + var scaledV = vImage_Buffer( + data: scaledVData, + height: vImagePixelCount(blurredHeight / 2), + width: vImagePixelCount(blurredWidth / 2), + rowBytes: blurredWidth / 2 + ) + + var scaledA = vImage_Buffer( + data: scaledAData, + height: vImagePixelCount(blurredHeight), + width: vImagePixelCount(blurredWidth), + rowBytes: blurredWidth + ) + + vImageScale_Planar8(&sourceY, &scaledY, nil, vImage_Flags(kvImageHighQualityResampling)) + vImageScale_Planar8(&sourceU, &scaledU, nil, vImage_Flags(kvImageHighQualityResampling)) + vImageScale_Planar8(&sourceV, &scaledV, nil, vImage_Flags(kvImageHighQualityResampling)) + vImageScale_Planar8(&sourceA, &scaledA, nil, vImage_Flags(kvImageHighQualityResampling)) + + guard let context = DrawingContext(size: size, scale: 1.0, clear: true) else { + return nil + } + + var destinationBuffer = vImage_Buffer( + data: context.bytes, + height: vImagePixelCount(blurredHeight), + width: vImagePixelCount(blurredWidth), + rowBytes: context.bytesPerRow + ) + + var result = kvImageNoError + + var permuteMap: [UInt8] = [1, 2, 3, 0] + result = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&scaledY, &scaledU, &scaledV, &destinationBuffer, &yuvToRgbConversion, &permuteMap, 255, vImage_Flags(kvImageDoNotTile)) + if result != kvImageNoError { + return nil + } + + result = vImageOverwriteChannels_ARGB8888(&scaledA, &destinationBuffer, &destinationBuffer, 1 << 0, vImage_Flags(kvImageDoNotTile)); + if result != kvImageNoError { + return nil + } + + vImageBoxConvolve_ARGB8888(&destinationBuffer, + &destinationBuffer, + nil, + 0, 0, + UInt32(15), + UInt32(15), + nil, + vImage_Flags(kvImageTruncateKernel)) + + let divisor: Int32 = 0x1000 + + let rwgt: CGFloat = 0.3086 + let gwgt: CGFloat = 0.6094 + let bwgt: CGFloat = 0.0820 + + let adjustSaturation: CGFloat = 1.7 + + let a = (1.0 - adjustSaturation) * rwgt + adjustSaturation + let b = (1.0 - adjustSaturation) * rwgt + let c = (1.0 - adjustSaturation) * rwgt + let d = (1.0 - adjustSaturation) * gwgt + let e = (1.0 - adjustSaturation) * gwgt + adjustSaturation + let f = (1.0 - adjustSaturation) * gwgt + let g = (1.0 - adjustSaturation) * bwgt + let h = (1.0 - adjustSaturation) * bwgt + let i = (1.0 - adjustSaturation) * bwgt + adjustSaturation + + let satMatrix: [CGFloat] = [ + a, b, c, 0, + d, e, f, 0, + g, h, i, 0, + 0, 0, 0, 1 + ] + + var matrix: [Int16] = satMatrix.map { value in + return Int16(value * CGFloat(divisor)) + } + + vImageMatrixMultiply_ARGB8888(&destinationBuffer, &destinationBuffer, &matrix, divisor, nil, nil, vImage_Flags(kvImageDoNotTile)) + + context.withFlippedContext { c in + c.setFillColor((color ?? .white).withMultipliedAlpha(0.6).cgColor) + c.fill(CGRect(origin: CGPoint(), size: size)) + } + + self.blurredRepresentationValue = context.generateImage() + return self.blurredRepresentationValue } } } @@ -230,6 +485,8 @@ private final class ItemAnimationContext { static let queue0 = Queue(name: "ItemAnimationContext-0", qos: .default) static let queue1 = Queue(name: "ItemAnimationContext-1", qos: .default) + private let useYuvA: Bool + private let cache: AnimationCache let queueAffinity: Int private let stateUpdated: () -> Void @@ -253,9 +510,10 @@ private final class ItemAnimationContext { let targets = Bag>() - init(cache: AnimationCache, queueAffinity: Int, itemId: String, size: CGSize, fetch: @escaping (AnimationCacheFetchOptions) -> Disposable, stateUpdated: @escaping () -> Void) { + init(cache: AnimationCache, queueAffinity: Int, itemId: String, size: CGSize, useYuvA: Bool, fetch: @escaping (AnimationCacheFetchOptions) -> Disposable, stateUpdated: @escaping () -> Void) { self.cache = cache self.queueAffinity = queueAffinity + self.useYuvA = useYuvA self.stateUpdated = stateUpdated self.disposable = cache.get(sourceId: itemId, size: size, fetch: fetch).start(next: { [weak self] result in @@ -300,7 +558,11 @@ private final class ItemAnimationContext { for target in self.targets.copyItems() { if let target = target.value { - target.transitionToContents(currentFrame.image.cgImage!, didLoop: false) + if let image = currentFrame.contentsAsImage { + target.transitionToContents(image.cgImage!, didLoop: false) + } else if let pixelBuffer = currentFrame.contentsAsCVPixelBuffer { + target.transitionToContents(pixelBuffer, didLoop: false) + } if let blurredRepresentationTarget = target.blurredRepresentationTarget { blurredRepresentationTarget.contents = currentFrame.blurredRepresentation(color: target.blurredRepresentationBackgroundColor)?.cgImage @@ -321,9 +583,15 @@ private final class ItemAnimationContext { func updateAddedTarget(target: MultiAnimationRenderTarget) { if let currentFrame = self.currentFrame { - if let cgImage = currentFrame.image.cgImage { + if let cgImage = currentFrame.contentsAsImage?.cgImage { target.transitionToContents(cgImage, didLoop: false) + if let blurredRepresentationTarget = target.blurredRepresentationTarget { + blurredRepresentationTarget.contents = currentFrame.blurredRepresentation(color: target.blurredRepresentationBackgroundColor)?.cgImage + } + } else if let pixelBuffer = currentFrame.contentsAsCVPixelBuffer { + target.transitionToContents(pixelBuffer, didLoop: false) + if let blurredRepresentationTarget = target.blurredRepresentationTarget { blurredRepresentationTarget.contents = currentFrame.blurredRepresentation(color: target.blurredRepresentationBackgroundColor)?.cgImage } @@ -388,12 +656,20 @@ private final class ItemAnimationContext { self.nextLoadingFrameTaskId += 1 self.loadingFrameTaskId = taskId + let useYuvA = self.useYuvA return LoadFrameGroupTask(task: { [weak self] in let currentFrame: (frame: Frame, didLoop: Bool)? do { if let (frame, didLoop) = try item.tryWith({ item -> (AnimationCacheItemFrame, Bool)? in - if let result = item.advance(advance: frameAdvance, requestedFormat: .rgba) { + let defaultFormat: AnimationCacheItemFrame.RequestedFormat + if useYuvA { + defaultFormat = .yuva(rowAlignment: 1) + } else { + defaultFormat = .rgba + } + + if let result = item.advance(advance: frameAdvance, requestedFormat: defaultFormat) { return (result.frame, result.didLoop) } else { return nil @@ -423,7 +699,11 @@ private final class ItemAnimationContext { strongSelf.currentFrame = currentFrame.frame for target in strongSelf.targets.copyItems() { if let target = target.value { - target.transitionToContents(currentFrame.frame.image.cgImage!, didLoop: currentFrame.didLoop) + if let image = currentFrame.frame.contentsAsImage { + target.transitionToContents(image.cgImage!, didLoop: currentFrame.didLoop) + } else if let pixelBuffer = currentFrame.frame.contentsAsCVPixelBuffer { + target.transitionToContents(pixelBuffer, didLoop: currentFrame.didLoop) + } if let blurredRepresentationTarget = target.blurredRepresentationTarget { blurredRepresentationTarget.contents = currentFrame.frame.blurredRepresentation(color: target.blurredRepresentationBackgroundColor)?.cgImage @@ -476,7 +756,7 @@ public final class MultiAnimationRendererImpl: MultiAnimationRenderer { self.stateUpdated = stateUpdated } - func add(target: MultiAnimationRenderTarget, cache: AnimationCache, itemId: String, unique: Bool, size: CGSize, fetch: @escaping (AnimationCacheFetchOptions) -> Disposable) -> Disposable { + func add(target: MultiAnimationRenderTarget, cache: AnimationCache, itemId: String, unique: Bool, size: CGSize, useYuvA: Bool, fetch: @escaping (AnimationCacheFetchOptions) -> Disposable) -> Disposable { var uniqueId = 0 if unique { uniqueId = self.nextUniqueId @@ -490,7 +770,7 @@ public final class MultiAnimationRendererImpl: MultiAnimationRenderer { } else { let queueAffinity = self.nextQueueAffinity self.nextQueueAffinity += 1 - itemContext = ItemAnimationContext(cache: cache, queueAffinity: queueAffinity, itemId: itemId, size: size, fetch: fetch, stateUpdated: { [weak self] in + itemContext = ItemAnimationContext(cache: cache, queueAffinity: queueAffinity, itemId: itemId, size: size, useYuvA: useYuvA, fetch: fetch, stateUpdated: { [weak self] in guard let strongSelf = self else { return } @@ -545,7 +825,11 @@ public final class MultiAnimationRendererImpl: MultiAnimationRenderer { return false } - target.contents = loadedFrame.image.cgImage + if let image = loadedFrame.contentsAsImage { + target.contents = image.cgImage + } else if let pixelBuffer = loadedFrame.contentsAsCVPixelBuffer { + target.contents = pixelBuffer + } target.numFrames = item.numFrames if let blurredRepresentationTarget = target.blurredRepresentationTarget { @@ -584,12 +868,18 @@ public final class MultiAnimationRendererImpl: MultiAnimationRenderer { } target.numFrames = item.numFrames if let loadedFrame = loadedFrame { - if let cgImage = loadedFrame.image.cgImage { + if let cgImage = loadedFrame.contentsAsImage?.cgImage { if hadIntermediateUpdate { target.transitionToContents(cgImage, didLoop: false) } else { target.contents = cgImage } + } else if let pixelBuffer = loadedFrame.contentsAsCVPixelBuffer { + if hadIntermediateUpdate { + target.transitionToContents(pixelBuffer, didLoop: false) + } else { + target.contents = pixelBuffer + } } if let blurredRepresentationTarget = target.blurredRepresentationTarget { @@ -622,8 +912,10 @@ public final class MultiAnimationRendererImpl: MultiAnimationRenderer { Queue.mainQueue().async { if let loadedFrame = loadedFrame { - if let cgImage = loadedFrame.image.cgImage { + if let cgImage = loadedFrame.contentsAsImage?.cgImage { completion(cgImage) + } else { + completion(nil) } } else { completion(nil) @@ -666,6 +958,7 @@ public final class MultiAnimationRendererImpl: MultiAnimationRenderer { public static let firstFrameQueue = Queue(name: "MultiAnimationRenderer-FirstFrame", qos: .userInteractive) + public var useYuvA: Bool = false private var groupContext: GroupContext? private var frameSkip: Int private var displayTimer: Foundation.Timer? @@ -728,7 +1021,7 @@ public final class MultiAnimationRendererImpl: MultiAnimationRenderer { self.groupContext = groupContext } - let disposable = groupContext.add(target: target, cache: cache, itemId: itemId, unique: unique, size: size, fetch: fetch) + let disposable = groupContext.add(target: target, cache: cache, itemId: itemId, unique: unique, size: size, useYuvA: self.useYuvA, fetch: fetch) return ActionDisposable { disposable.dispose() From 97c02dd5b567a7028d0e4644514e700adadcdd68 Mon Sep 17 00:00:00 2001 From: Isaac <> Date: Fri, 14 Mar 2025 15:34:31 +0100 Subject: [PATCH 3/5] IOSurface experiment --- submodules/TelegramUI/Sources/AccountContext.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/submodules/TelegramUI/Sources/AccountContext.swift b/submodules/TelegramUI/Sources/AccountContext.swift index 2d8d84761f..38f9c18762 100644 --- a/submodules/TelegramUI/Sources/AccountContext.swift +++ b/submodules/TelegramUI/Sources/AccountContext.swift @@ -319,6 +319,7 @@ public final class AccountContextImpl: AccountContext { } }) self.animationRenderer = MultiAnimationRendererImpl() + (self.animationRenderer as? MultiAnimationRendererImpl)?.useYuvA = sharedContext.immediateExperimentalUISettings.compressedEmojiCache let updatedLimitsConfiguration = account.postbox.preferencesView(keys: [PreferencesKeys.limitsConfiguration]) |> map { preferences -> LimitsConfiguration in @@ -452,6 +453,17 @@ public final class AccountContextImpl: AccountContext { } self.audioTranscriptionTrial = audioTranscriptionTrial }) + + self.experimentalUISettingsDisposable = (sharedContext.accountManager.sharedData(keys: [ApplicationSpecificSharedDataKeys.experimentalUISettings]) + |> deliverOnMainQueue).start(next: { [weak self] sharedData in + guard let self else { + return + } + guard let settings = sharedData.entries[ApplicationSpecificSharedDataKeys.experimentalUISettings]?.get(ExperimentalUISettings.self) else { + return + } + (self.animationRenderer as? MultiAnimationRendererImpl)?.useYuvA = settings.compressedEmojiCache + }) } deinit { From 1304f0f098b113b45f70d62ec9a7a64755903ca9 Mon Sep 17 00:00:00 2001 From: Isaac <> Date: Fri, 14 Mar 2025 15:34:45 +0100 Subject: [PATCH 4/5] IOSurface experiment --- .../Sources/ExperimentalUISettings.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/submodules/TelegramUIPreferences/Sources/ExperimentalUISettings.swift b/submodules/TelegramUIPreferences/Sources/ExperimentalUISettings.swift index 3a2400cd6d..18beed096d 100644 --- a/submodules/TelegramUIPreferences/Sources/ExperimentalUISettings.swift +++ b/submodules/TelegramUIPreferences/Sources/ExperimentalUISettings.swift @@ -38,7 +38,7 @@ public struct ExperimentalUISettings: Codable, Equatable { public var experimentalCompatibility: Bool public var enableDebugDataDisplay: Bool public var rippleEffect: Bool - public var inlineStickers: Bool + public var compressedEmojiCache: Bool public var localTranscription: Bool public var enableReactionOverrides: Bool public var browserExperiment: Bool @@ -81,7 +81,7 @@ public struct ExperimentalUISettings: Codable, Equatable { experimentalCompatibility: false, enableDebugDataDisplay: false, rippleEffect: false, - inlineStickers: false, + compressedEmojiCache: false, localTranscription: false, enableReactionOverrides: false, browserExperiment: false, @@ -125,7 +125,7 @@ public struct ExperimentalUISettings: Codable, Equatable { experimentalCompatibility: Bool, enableDebugDataDisplay: Bool, rippleEffect: Bool, - inlineStickers: Bool, + compressedEmojiCache: Bool, localTranscription: Bool, enableReactionOverrides: Bool, browserExperiment: Bool, @@ -166,7 +166,7 @@ public struct ExperimentalUISettings: Codable, Equatable { self.experimentalCompatibility = experimentalCompatibility self.enableDebugDataDisplay = enableDebugDataDisplay self.rippleEffect = rippleEffect - self.inlineStickers = inlineStickers + self.compressedEmojiCache = compressedEmojiCache self.localTranscription = localTranscription self.enableReactionOverrides = enableReactionOverrides self.browserExperiment = browserExperiment @@ -211,7 +211,7 @@ public struct ExperimentalUISettings: Codable, Equatable { self.experimentalCompatibility = (try container.decodeIfPresent(Int32.self, forKey: "experimentalCompatibility") ?? 0) != 0 self.enableDebugDataDisplay = (try container.decodeIfPresent(Int32.self, forKey: "enableDebugDataDisplay") ?? 0) != 0 self.rippleEffect = (try container.decodeIfPresent(Int32.self, forKey: "rippleEffect") ?? 0) != 0 - self.inlineStickers = (try container.decodeIfPresent(Int32.self, forKey: "inlineStickers") ?? 0) != 0 + self.compressedEmojiCache = (try container.decodeIfPresent(Int32.self, forKey: "compressedEmojiCache") ?? 0) != 0 self.localTranscription = (try container.decodeIfPresent(Int32.self, forKey: "localTranscription") ?? 0) != 0 self.enableReactionOverrides = try container.decodeIfPresent(Bool.self, forKey: "enableReactionOverrides") ?? false self.browserExperiment = try container.decodeIfPresent(Bool.self, forKey: "browserExperiment") ?? false @@ -256,7 +256,7 @@ public struct ExperimentalUISettings: Codable, Equatable { try container.encode((self.experimentalCompatibility ? 1 : 0) as Int32, forKey: "experimentalCompatibility") try container.encode((self.enableDebugDataDisplay ? 1 : 0) as Int32, forKey: "enableDebugDataDisplay") try container.encode((self.rippleEffect ? 1 : 0) as Int32, forKey: "rippleEffect") - try container.encode((self.inlineStickers ? 1 : 0) as Int32, forKey: "inlineStickers") + try container.encode((self.compressedEmojiCache ? 1 : 0) as Int32, forKey: "compressedEmojiCache") try container.encode((self.localTranscription ? 1 : 0) as Int32, forKey: "localTranscription") try container.encode(self.enableReactionOverrides, forKey: "enableReactionOverrides") try container.encode(self.browserExperiment, forKey: "browserExperiment") From 13488b2eaab1180522038567aacc8f81882e775b Mon Sep 17 00:00:00 2001 From: Isaac <> Date: Fri, 14 Mar 2025 15:36:23 +0100 Subject: [PATCH 5/5] VoIP updates --- .../Sources/OngoingCallThreadLocalContext.mm | 70 +++++++++++++++---- submodules/TgVoipWebrtc/tgcalls | 2 +- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm b/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm index b07b39824b..7ff25040d7 100644 --- a/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm +++ b/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm @@ -1,4 +1,5 @@ #import +#include #import "MediaUtils.h" @@ -6,7 +7,6 @@ #import "InstanceImpl.h" #import "v2/InstanceV2Impl.h" #import "v2/InstanceV2ReferenceImpl.h" -//#import "v2_4_0_0/InstanceV2_4_0_0Impl.h" #include "StaticThreads.h" #import "VideoCaptureInterface.h" @@ -507,8 +507,8 @@ public: ) override { _mutex.Lock(); if (!_audioTransports.empty()) { - for (size_t i = _audioTransports.size() - 1; i < _audioTransports.size(); i++) { - _audioTransports[_audioTransports.size() - 1]->RecordedDataIsAvailable( + for (size_t i = 0; i < _audioTransports.size(); i++) { + _audioTransports[i]->RecordedDataIsAvailable( audioSamples, nSamples, nBytesPerSample, @@ -542,16 +542,59 @@ public: int32_t result = 0; if (!_audioTransports.empty()) { - result = _audioTransports[_audioTransports.size() - 1]->NeedMorePlayData( - nSamples, - nBytesPerSample, - nChannels, - samplesPerSec, - audioSamples, - nSamplesOut, - elapsed_time_ms, - ntp_time_ms - ); + if (_audioTransports.size() > 1) { + size_t totalNumSamples = nSamples * nBytesPerSample * nChannels; + if (_mixAudioSamples.size() < totalNumSamples) { + _mixAudioSamples.resize(totalNumSamples); + } + memset(audioSamples, 0, totalNumSamples); + + int16_t *resultAudioSamples = (int16_t *)audioSamples; + + for (size_t i = 0; i < _audioTransports.size(); i++) { + int64_t localElapsedTimeMs = 0; + int64_t localNtpTimeMs = 0; + size_t localNSamplesOut = 0; + + _audioTransports[i]->NeedMorePlayData( + nSamples, + nBytesPerSample, + nChannels, + samplesPerSec, + _mixAudioSamples.data(), + localNSamplesOut, + &localElapsedTimeMs, + &localNtpTimeMs + ); + + for (size_t j = 0; j < localNSamplesOut; j++) { + int32_t mixedSample = (int32_t)resultAudioSamples[j] + (int32_t)_mixAudioSamples[j]; + resultAudioSamples[j] = (int16_t)std::clamp(mixedSample, INT16_MIN, INT16_MAX); + } + + if (i == _audioTransports.size() - 1) { + nSamplesOut = localNSamplesOut; + if (elapsed_time_ms) { + *elapsed_time_ms = localElapsedTimeMs; + } + if (ntp_time_ms) { + *ntp_time_ms = localNtpTimeMs; + } + } + } + nSamplesOut = nSamples; + } else { + result = _audioTransports[_audioTransports.size() - 1]->NeedMorePlayData( + nSamples, + nBytesPerSample, + nChannels, + samplesPerSec, + audioSamples, + nSamplesOut, + elapsed_time_ms, + ntp_time_ms + ); + } } else { nSamplesOut = 0; } @@ -620,6 +663,7 @@ private: bool _isStarted = false; std::vector _audioTransports; webrtc::Mutex _mutex; + std::vector _mixAudioSamples; }; class WrappedChildAudioDeviceModule : public tgcalls::DefaultWrappedAudioDeviceModule { diff --git a/submodules/TgVoipWebrtc/tgcalls b/submodules/TgVoipWebrtc/tgcalls index 4d122958a6..6f4e6f1b4c 160000 --- a/submodules/TgVoipWebrtc/tgcalls +++ b/submodules/TgVoipWebrtc/tgcalls @@ -1 +1 @@ -Subproject commit 4d122958a6bd8aa94437cf26750442b88cc0f5c0 +Subproject commit 6f4e6f1b4cdfed02a30a2883a8ecb7586732ddfc