From 031fe813262c12c64367da0ead9954b5e14c1910 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 15:51:08 +0500 Subject: [PATCH 1/8] Temp --- .../Sources/ChatImportActivityScreen.swift | 363 +++++++++++------- 1 file changed, 219 insertions(+), 144 deletions(-) diff --git a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift index 46891b2e4d..65c3a1be96 100644 --- a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift +++ b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift @@ -50,19 +50,201 @@ private final class ProgressEstimator { } } -public final class ChatImportActivityScreen: ViewController { +private final class ImportManager { enum ImportError { case generic case chatAdminRequired case invalidChatType } - private enum State { - case progress(CGFloat) + enum State { + case progress(totalBytes: Int, totalUploadedBytes: Int) case error(ImportError) case done } + private let account: Account + private let archivePath: String + private let entries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] + + private var session: ChatHistoryImport.Session? + + private let disposable = MetaDisposable() + + private let totalBytes: Int + private var pendingEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] + private var entryProgress: [String: (Int, Int)] = [:] + private var activeEntries: [String: Disposable] = [:] + + private var stateValue: State { + didSet { + self.statePromise.set(.single(self.stateValue)) + } + } + private let statePromise = Promise() + var state: Signal { + return self.statePromise.get() + } + + init(account: Account, peerId: PeerId, mainFile: TempBoxFile, archivePath: String, entries: [(SSZipEntry, String, ChatHistoryImport.MediaType)]) { + self.account = account + self.archivePath = archivePath + self.entries = entries + self.pendingEntries = entries + + var totalBytes = 0 + for entry in self.entries { + self.entryProgress[entry.0.path] = (Int(entry.0.uncompressedSize), 0) + totalBytes += Int(entry.0.uncompressedSize) + } + self.totalBytes = totalBytes + + self.stateValue = .progress(totalBytes: totalBytes, totalUploadedBytes: 0) + + self.disposable.set((ChatHistoryImport.initSession(account: self.account, peerId: peerId, file: mainFile, mediaCount: Int32(entries.count)) + |> mapError { error -> ImportError in + switch error { + case .chatAdminRequired: + return .chatAdminRequired + case .invalidChatType: + return .invalidChatType + case .generic: + return .generic + } + } + |> deliverOnMainQueue).start(next: { [weak self] session in + guard let strongSelf = self else { + return + } + strongSelf.session = session + strongSelf.updateState() + }, error: { [weak self] error in + guard let strongSelf = self else { + return + } + strongSelf.failWithError(error) + })) + } + + deinit { + self.disposable.dispose() + for (_, disposable) in self.activeEntries { + disposable.dispose() + } + } + + private func updateProgress() { + if case .error = self.stateValue { + return + } + + var totalSize = 0 + var totalUploadedSize = 0 + for (_, entrySizes) in self.entryProgress { + totalSize += entrySizes.0 + totalUploadedSize += entrySizes.1 + } + + self.stateValue = .progress(totalBytes: self.totalBytes, totalUploadedBytes: totalUploadedSize) + } + + private func failWithError(_ error: ImportError) { + self.stateValue = .error(error) + for (_, disposable) in self.activeEntries { + disposable.dispose() + } + } + + private func complete() { + guard let session = self.session else { + self.failWithError(.generic) + } + self.disposable.set((ChatHistoryImport.startImport(account: self.account, session: session) + |> deliverOnMainQueue).start(error: { [weak self] _ in + guard let strongSelf = self else { + return + } + strongSelf.failWithError(.generic) + }, completed: { [weak self] in + guard let strongSelf = self else { + return + } + strongSelf.stateValue = .done + })) + } + + private func updateState() { + guard let session = self.session else { + return + } + if self.activeEntries.count >= 2 { + return + } + if self.pendingEntries.isEmpty { + self.complete() + return + } + if case .error = self.stateValue { + return + } + + let entry = self.pendingEntries.removeFirst() + let archivePath = self.archivePath + let unpackedFile = Signal { subscriber in + let tempFile = TempBox.shared.tempFile(fileName: entry.0.path) + print("Extracting \(entry.0.path) to \(tempFile.path)...") + let startTime = CACurrentMediaTime() + if SSZipArchive.extractFileFromArchive(atPath: archivePath, filePath: entry.0.path, toPath: tempFile.path) { + print("[Done in \(CACurrentMediaTime() - startTime) s] Extract \(entry.0.path) to \(tempFile.path)") + subscriber.putNext(tempFile) + subscriber.putCompletion() + } else { + subscriber.putError(.generic) + } + + return EmptyDisposable + } + + let account = self.account + + let uploadedEntrySignal: Signal = unpackedFile + |> mapToSignal { tempFile -> Signal in + return ChatHistoryImport.uploadMedia(account: account, session: session, file: tempFile, fileName: entry.0.path, mimeType: entry.1, type: entry.2) + |> mapError { error -> ImportError in + switch error { + case .chatAdminRequired: + return .chatAdminRequired + case .generic: + return .generic + } + } + } + + self.activeEntries[entry.1] = (uploadedEntrySignal + |> deliverOnMainQueue).start(next: { [weak self] progress in + guard let strongSelf = self else { + return + } + if let (size, _) = strongSelf.entryProgress[entry.0.path] { + strongSelf.entryProgress[entry.0.path] = (size, Int(progress * Float(entry.0.uncompressedSize))) + strongSelf.updateProgress() + } + }, error: { [weak self] error in + guard let strongSelf = self else { + return + } + strongSelf.failWithError(error) + }, completed: { [weak self] in + guard let strongSelf = self else { + return + } + strongSelf.activeEntries.removeValue(forKey: entry.0.path) + strongSelf.updateState() + }) + } +} + +public final class ChatImportActivityScreen: ViewController { private final class Node: ViewControllerTracingNode { private weak var controller: ChatImportActivityScreen? @@ -85,7 +267,7 @@ public final class ChatImportActivityScreen: ViewController { private var validLayout: (ContainerViewLayout, CGFloat)? private let totalBytes: Int - private var state: State = .progress(0.0) + private var state: ImportManager.State private var videoNode: UniversalVideoNode? private var feedback: HapticFeedback? @@ -96,6 +278,7 @@ public final class ChatImportActivityScreen: ViewController { self.controller = controller self.context = context self.totalBytes = totalBytes + self.state = .progress(totalBytes: totalBytes, totalUploadedBytes: 0) self.presentationData = self.context.sharedContext.currentPresentationData.with { $0 } @@ -254,8 +437,12 @@ public final class ChatImportActivityScreen: ViewController { let effectiveProgress: CGFloat switch state { - case let .progress(value): - effectiveProgress = value + case let .progress(totalBytes, totalUploadedBytes): + if totalBytes == 0 { + effectiveProgress = 1.0 + } else { + effectiveProgress = CGFloat(totalUploadedBytes) / CGFloat(totalBytes) + } case .error: effectiveProgress = 0.0 case .done: @@ -380,7 +567,7 @@ public final class ChatImportActivityScreen: ViewController { self.animationNode.stopAtNearestLoop = true } - func updateState(state: State, animated: Bool) { + func updateState(state: ImportManager.State, animated: Bool) { var wasDone = false if case .done = self.state { wasDone = true @@ -392,8 +579,12 @@ public final class ChatImportActivityScreen: ViewController { let effectiveProgress: CGFloat switch state { - case let .progress(value): - effectiveProgress = value + case let .progress(totalBytes, totalUploadedBytes): + if totalBytes == 0 { + effectiveProgress = 1.0 + } else { + effectiveProgress = CGFloat(totalUploadedBytes) / CGFloat(totalBytes) + } case .error: effectiveProgress = 0.0 case .done: @@ -454,10 +645,9 @@ public final class ChatImportActivityScreen: ViewController { private let archivePath: String private let mainEntry: TempBoxFile private let mainEntrySize: Int - private let otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType, Signal)] - private let totalBytes: Int - private let totalMediaBytes: Int + private let otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] + private var importManager: ImportManager? private var progressEstimator: ProgressEstimator? private var totalMediaProgress: Float = 0.0 private var beganCompletion: Bool = false @@ -465,6 +655,7 @@ public final class ChatImportActivityScreen: ViewController { private var pendingEntries: [String: (Int, Float)] = [:] private let disposable = MetaDisposable() + private let progressDisposable = MetaDisposable() override public var _presentedInModal: Bool { get { @@ -480,26 +671,8 @@ public final class ChatImportActivityScreen: ViewController { self.archivePath = archivePath self.mainEntry = mainEntry - var isFirstFile = true - self.otherEntries = otherEntries.map { entry -> (SSZipEntry, String, ChatHistoryImport.MediaType, Signal) in - let signal = Signal { subscriber in - let tempFile = TempBox.shared.tempFile(fileName: entry.1) - print("Extracting \(entry.0.path) to \(tempFile.path)...") - let startTime = CACurrentMediaTime() - if SSZipArchive.extractFileFromArchive(atPath: archivePath, filePath: entry.0.path, toPath: tempFile.path) { - print("[Done in \(CACurrentMediaTime() - startTime) s] Extract \(entry.0.path) to \(tempFile.path)") - subscriber.putNext(tempFile) - subscriber.putCompletion() - } else { - subscriber.putNext(nil) - subscriber.putCompletion() - } - - return EmptyDisposable - } - //let promise = Promise() - //promise.set(signal) - return (entry.0, entry.1, entry.2, signal) + self.otherEntries = otherEntries.map { entry -> (SSZipEntry, String, ChatHistoryImport.MediaType) in + return (entry.0, entry.1, entry.2) } if let size = fileSize(self.mainEntry.path) { @@ -516,8 +689,8 @@ public final class ChatImportActivityScreen: ViewController { for entry in self.otherEntries { totalMediaBytes += Int(entry.0.uncompressedSize) } - self.totalBytes = self.mainEntrySize + totalMediaBytes - self.totalMediaBytes = totalMediaBytes + //self.totalBytes = self.mainEntrySize + totalMediaBytes + //self.totalMediaBytes = totalMediaBytes self.presentationData = self.context.sharedContext.currentPresentationData.with { $0 } @@ -544,6 +717,7 @@ public final class ChatImportActivityScreen: ViewController { deinit { self.disposable.dispose() + self.progressDisposable.dispose() if let application = UIApplication.value(forKeyPath: #keyPath(UIApplication.shared)) as? UIApplication { application.isIdleTimerDisabled = false @@ -574,16 +748,10 @@ public final class ChatImportActivityScreen: ViewController { self.progressEstimator = ProgressEstimator() self.beganCompletion = false - self.controllerNode.updateState(state: .progress(0.0), animated: true) - - let context = self.context - let mainEntry = self.mainEntry - let otherEntries = self.otherEntries - - let resolvedPeerId: Signal + let resolvedPeerId: Signal if self.peerId.namespace == Namespaces.Peer.CloudGroup { resolvedPeerId = convertGroupToSupergroup(account: self.context.account, peerId: self.peerId) - |> mapError { _ -> ImportError in + |> mapError { _ -> ImportManager.ImportError in return .generic } } else { @@ -591,117 +759,24 @@ public final class ChatImportActivityScreen: ViewController { } self.disposable.set((resolvedPeerId - |> mapToSignal { [weak self] peerId -> Signal in - Queue.mainQueue().async { - self?.peerId = peerId - } - - return ChatHistoryImport.initSession(account: context.account, peerId: peerId, file: mainEntry, mediaCount: Int32(otherEntries.count)) - |> mapError { error -> ImportError in - switch error { - case .chatAdminRequired: - return .chatAdminRequired - case .invalidChatType: - return .invalidChatType - case .generic: - return .generic - } - } - } - |> mapToSignal { session -> Signal<[(String, Float)], ImportError> in - var mediaSignals: [Signal<(String, Float), ImportError>] = [] - - for (_, fileName, mediaType, fileData) in otherEntries { - let unpackedFile: Signal = fileData - |> take(1) - |> deliverOnMainQueue - |> castError(ImportError.self) - |> mapToSignal { file -> Signal in - if let file = file { - return .single(file) - } else { - return .fail(.generic) - } - } - let uploadedMedia = unpackedFile - |> mapToSignal { tempFile -> Signal<(String, Float), ImportError> in - var mimeTypeValue = "application/binary" - let fileExtension = (tempFile.path as NSString).pathExtension - if !fileExtension.isEmpty { - if let value = TGMimeTypeMap.mimeType(forExtension: fileExtension.lowercased()) { - mimeTypeValue = value - } - } - - return ChatHistoryImport.uploadMedia(account: context.account, session: session, file: tempFile, fileName: fileName, mimeType: mimeTypeValue, type: mediaType) - |> mapError { error -> ImportError in - switch error { - case .chatAdminRequired: - return .chatAdminRequired - case .generic: - return .generic - } - } - |> map { progress -> (String, Float) in - return (fileName, progress) - } - } - - mediaSignals.append(Signal<(String, Float), ImportError>.single((fileName, 0.0)) - |> then(uploadedMedia)) - } - - return combineLatest(mediaSignals) - |> then(ChatHistoryImport.startImport(account: context.account, session: session) - |> mapError { _ -> ImportError in - return .generic - } - |> map { _ -> [(String, Float)] in - }) - } - |> deliverOnMainQueue).start(next: { [weak self] fileNameAndProgress in + |> deliverOnMainQueue).start(next: { [weak self] peerId in guard let strongSelf = self else { return } - - for (fileName, progress) in fileNameAndProgress { - if let (fileSize, _) = strongSelf.pendingEntries[fileName] { - strongSelf.pendingEntries[fileName] = (fileSize, progress) + let importManager = ImportManager(account: strongSelf.context.account, peerId: peerId, mainFile: strongSelf.mainEntry, archivePath: strongSelf.archivePath, entries: strongSelf.otherEntries) + strongSelf.importManager = importManager + strongSelf.progressDisposable.set((importManager.state + |> deliverOnMainQueue).start(next: { state in + guard let strongSelf = self else { + return } - } - - var totalDoneMediaBytes = 0 - for (_, sizeAndProgress) in strongSelf.pendingEntries { - totalDoneMediaBytes += Int(Float(sizeAndProgress.0) * sizeAndProgress.1) - } - - let totalDoneBytes = strongSelf.mainEntrySize + totalDoneMediaBytes - - var totalProgress: CGFloat = 1.0 - if !strongSelf.otherEntries.isEmpty { - totalProgress = CGFloat(totalDoneBytes) / CGFloat(strongSelf.totalBytes) - } - var totalMediaProgress: CGFloat = 1.0 - if !strongSelf.otherEntries.isEmpty { - totalProgress = CGFloat(totalDoneBytes) / CGFloat(strongSelf.totalBytes) - totalMediaProgress = CGFloat(totalDoneMediaBytes) / CGFloat(strongSelf.totalMediaBytes) - } - strongSelf.controllerNode.updateState(state: .progress(totalProgress), animated: true) - strongSelf.totalMediaProgress = Float(totalMediaProgress) + strongSelf.controllerNode.updateState(state: state, animated: true) + })) }, error: { [weak self] error in guard let strongSelf = self else { return } strongSelf.controllerNode.updateState(state: .error(error), animated: true) - }, completed: { [weak self] in - guard let strongSelf = self else { - return - } - strongSelf.controllerNode.updateState(state: .done, animated: true) - - if let application = UIApplication.value(forKeyPath: #keyPath(UIApplication.shared)) as? UIApplication { - application.isIdleTimerDisabled = false - } })) } From c27376b51535fd0f400b4b7f4a16b9a63fe70eba Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 16:41:17 +0500 Subject: [PATCH 2/8] Don't display notifications for imported messages --- submodules/TelegramUI/Sources/ApplicationContext.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/submodules/TelegramUI/Sources/ApplicationContext.swift b/submodules/TelegramUI/Sources/ApplicationContext.swift index 1ac1a97b48..71fa65ac28 100644 --- a/submodules/TelegramUI/Sources/ApplicationContext.swift +++ b/submodules/TelegramUI/Sources/ApplicationContext.swift @@ -319,6 +319,9 @@ final class AuthorizedApplicationContext { } } } + if let forwardInfo = firstMessage.forwardInfo, forwardInfo.flags.contains(.isImported) { + return + } if chatIsVisible { return From 127b41a959a8e5e1ba777866f1aeca0dcf05908a Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 17:17:32 +0500 Subject: [PATCH 3/8] Rewrite import upload --- .../Sources/ChatImportActivityScreen.swift | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift index 65c3a1be96..020ae7aa7c 100644 --- a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift +++ b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift @@ -29,7 +29,7 @@ private final class ProgressEstimator { if let (lastTimestamp, lastProgress) = self.lastMeasurement { if abs(lastProgress - progress) >= 0.01 || abs(lastTimestamp - timestamp) > 1.0 { let immediateProgressPerSecond = Double(progress - lastProgress) / (timestamp - lastTimestamp) - let alpha: Double = 0.05 + let alpha: Double = 0.01 self.averageProgressPerSecond = alpha * immediateProgressPerSecond + (1.0 - alpha) * self.averageProgressPerSecond self.lastMeasurement = (timestamp, progress) } @@ -45,6 +45,7 @@ private final class ProgressEstimator { } else { let remainingProgress = Double(1.0 - progress) let remainingTime = remainingProgress / self.averageProgressPerSecond + //print("remainingTime \(remainingTime)") return remainingTime } } @@ -58,7 +59,7 @@ private final class ImportManager { } enum State { - case progress(totalBytes: Int, totalUploadedBytes: Int) + case progress(totalBytes: Int, totalUploadedBytes: Int, totalMediaBytes: Int, totalUploadedMediaBytes: Int) case error(ImportError) case done } @@ -72,6 +73,8 @@ private final class ImportManager { private let disposable = MetaDisposable() private let totalBytes: Int + private let totalMediaBytes: Int + private let mainFileSize: Int private var pendingEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] private var entryProgress: [String: (Int, Int)] = [:] private var activeEntries: [String: Disposable] = [:] @@ -92,14 +95,17 @@ private final class ImportManager { self.entries = entries self.pendingEntries = entries - var totalBytes = 0 + self.mainFileSize = fileSize(mainFile.path) ?? 0 + + var totalMediaBytes = 0 for entry in self.entries { self.entryProgress[entry.0.path] = (Int(entry.0.uncompressedSize), 0) - totalBytes += Int(entry.0.uncompressedSize) + totalMediaBytes += Int(entry.0.uncompressedSize) } - self.totalBytes = totalBytes + self.totalBytes = self.mainFileSize + totalMediaBytes + self.totalMediaBytes = totalMediaBytes - self.stateValue = .progress(totalBytes: totalBytes, totalUploadedBytes: 0) + self.stateValue = .progress(totalBytes: self.totalBytes, totalUploadedBytes: 0, totalMediaBytes: self.totalMediaBytes, totalUploadedMediaBytes: 0) self.disposable.set((ChatHistoryImport.initSession(account: self.account, peerId: peerId, file: mainFile, mediaCount: Int32(entries.count)) |> mapError { error -> ImportError in @@ -138,14 +144,17 @@ private final class ImportManager { return } - var totalSize = 0 - var totalUploadedSize = 0 + var totalUploadedMediaBytes = 0 for (_, entrySizes) in self.entryProgress { - totalSize += entrySizes.0 - totalUploadedSize += entrySizes.1 + totalUploadedMediaBytes += entrySizes.1 } - self.stateValue = .progress(totalBytes: self.totalBytes, totalUploadedBytes: totalUploadedSize) + var totalUploadedBytes = totalUploadedMediaBytes + if let _ = self.session { + totalUploadedBytes += self.mainFileSize + } + + self.stateValue = .progress(totalBytes: self.totalBytes, totalUploadedBytes: totalUploadedBytes, totalMediaBytes: self.totalMediaBytes, totalUploadedMediaBytes: totalUploadedMediaBytes) } private func failWithError(_ error: ImportError) { @@ -158,6 +167,7 @@ private final class ImportManager { private func complete() { guard let session = self.session else { self.failWithError(.generic) + return } self.disposable.set((ChatHistoryImport.startImport(account: self.account, session: session) |> deliverOnMainQueue).start(error: { [weak self] _ in @@ -177,7 +187,7 @@ private final class ImportManager { guard let session = self.session else { return } - if self.activeEntries.count >= 2 { + if self.activeEntries.count >= 3 { return } if self.pendingEntries.isEmpty { @@ -274,11 +284,11 @@ public final class ChatImportActivityScreen: ViewController { fileprivate var remainingAnimationSeconds: Double? - init(controller: ChatImportActivityScreen, context: AccountContext, totalBytes: Int) { + init(controller: ChatImportActivityScreen, context: AccountContext, totalBytes: Int, totalMediaBytes: Int) { self.controller = controller self.context = context self.totalBytes = totalBytes - self.state = .progress(totalBytes: totalBytes, totalUploadedBytes: 0) + self.state = .progress(totalBytes: totalBytes, totalUploadedBytes: 0, totalMediaBytes: totalMediaBytes, totalUploadedMediaBytes: 0) self.presentationData = self.context.sharedContext.currentPresentationData.with { $0 } @@ -437,7 +447,7 @@ public final class ChatImportActivityScreen: ViewController { let effectiveProgress: CGFloat switch state { - case let .progress(totalBytes, totalUploadedBytes): + case let .progress(totalBytes, totalUploadedBytes, _, _): if totalBytes == 0 { effectiveProgress = 1.0 } else { @@ -579,7 +589,7 @@ public final class ChatImportActivityScreen: ViewController { let effectiveProgress: CGFloat switch state { - case let .progress(totalBytes, totalUploadedBytes): + case let .progress(totalBytes, totalUploadedBytes, _, _): if totalBytes == 0 { effectiveProgress = 1.0 } else { @@ -644,7 +654,8 @@ public final class ChatImportActivityScreen: ViewController { fileprivate var peerId: PeerId private let archivePath: String private let mainEntry: TempBoxFile - private let mainEntrySize: Int + private let totalBytes: Int + private let totalMediaBytes: Int private let otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] private var importManager: ImportManager? @@ -652,8 +663,6 @@ public final class ChatImportActivityScreen: ViewController { private var totalMediaProgress: Float = 0.0 private var beganCompletion: Bool = false - private var pendingEntries: [String: (Int, Float)] = [:] - private let disposable = MetaDisposable() private let progressDisposable = MetaDisposable() @@ -675,22 +684,14 @@ public final class ChatImportActivityScreen: ViewController { return (entry.0, entry.1, entry.2) } - if let size = fileSize(self.mainEntry.path) { - self.mainEntrySize = size - } else { - self.mainEntrySize = 0 - } - - for (entry, fileName, _) in otherEntries { - self.pendingEntries[fileName] = (Int(entry.uncompressedSize), 0.0) - } + let mainEntrySize = fileSize(self.mainEntry.path) ?? 0 var totalMediaBytes = 0 for entry in self.otherEntries { totalMediaBytes += Int(entry.0.uncompressedSize) } - //self.totalBytes = self.mainEntrySize + totalMediaBytes - //self.totalMediaBytes = totalMediaBytes + self.totalBytes = mainEntrySize + totalMediaBytes + self.totalMediaBytes = totalMediaBytes self.presentationData = self.context.sharedContext.currentPresentationData.with { $0 } @@ -729,7 +730,7 @@ public final class ChatImportActivityScreen: ViewController { } override public func loadDisplayNode() { - self.displayNode = Node(controller: self, context: self.context, totalBytes: self.totalBytes) + self.displayNode = Node(controller: self, context: self.context, totalBytes: self.totalBytes, totalMediaBytes: self.totalMediaBytes) self.displayNodeDidLoad() } @@ -741,10 +742,6 @@ public final class ChatImportActivityScreen: ViewController { } private func beginImport() { - for (key, value) in self.pendingEntries { - self.pendingEntries[key] = (value.0, 0.0) - } - self.progressEstimator = ProgressEstimator() self.beganCompletion = false @@ -771,6 +768,12 @@ public final class ChatImportActivityScreen: ViewController { return } strongSelf.controllerNode.updateState(state: state, animated: true) + if case let .progress(_, _, totalMediaBytes, totalUploadedMediaBytes) = state { + if let progressEstimator = strongSelf.progressEstimator { + let progress = Float(totalUploadedMediaBytes) / Float(totalMediaBytes) + strongSelf.totalMediaProgress = progress + } + } })) }, error: { [weak self] error in guard let strongSelf = self else { From ff7446ba697775e5a7ac13d649967a65c8372df0 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 18:40:22 +0500 Subject: [PATCH 4/8] Chat import fixes --- .../Sources/ChatImportActivityScreen.swift | 129 +++++++------- .../PublicHeaders/MtProtoKit/MTGzip.h | 1 + submodules/MtProtoKit/Sources/MTGzip.m | 38 +++++ .../Sources/ChatHistoryImport.swift | 2 +- .../TelegramCore/Sources/Download.swift | 24 ++- .../Sources/MultipartUpload.swift | 18 +- .../Sources/ShareExtensionContext.swift | 161 ++++++++++-------- 7 files changed, 230 insertions(+), 143 deletions(-) diff --git a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift index 020ae7aa7c..eead72c488 100644 --- a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift +++ b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift @@ -65,7 +65,7 @@ private final class ImportManager { } private let account: Account - private let archivePath: String + private let archivePath: String? private let entries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] private var session: ChatHistoryImport.Session? @@ -89,7 +89,7 @@ private final class ImportManager { return self.statePromise.get() } - init(account: Account, peerId: PeerId, mainFile: TempBoxFile, archivePath: String, entries: [(SSZipEntry, String, ChatHistoryImport.MediaType)]) { + init(account: Account, peerId: PeerId, mainFile: TempBoxFile, archivePath: String?, entries: [(SSZipEntry, String, ChatHistoryImport.MediaType)]) { self.account = account self.archivePath = archivePath self.entries = entries @@ -187,70 +187,81 @@ private final class ImportManager { guard let session = self.session else { return } - if self.activeEntries.count >= 3 { - return - } - if self.pendingEntries.isEmpty { + if self.pendingEntries.isEmpty && self.activeEntries.isEmpty { self.complete() return } if case .error = self.stateValue { return } + guard let archivePath = self.archivePath else { + return + } - let entry = self.pendingEntries.removeFirst() - let archivePath = self.archivePath - let unpackedFile = Signal { subscriber in - let tempFile = TempBox.shared.tempFile(fileName: entry.0.path) - print("Extracting \(entry.0.path) to \(tempFile.path)...") - let startTime = CACurrentMediaTime() - if SSZipArchive.extractFileFromArchive(atPath: archivePath, filePath: entry.0.path, toPath: tempFile.path) { - print("[Done in \(CACurrentMediaTime() - startTime) s] Extract \(entry.0.path) to \(tempFile.path)") - subscriber.putNext(tempFile) - subscriber.putCompletion() - } else { - subscriber.putError(.generic) + while true { + if self.activeEntries.count >= 3 { + break + } + if self.pendingEntries.isEmpty { + break } - return EmptyDisposable - } - - let account = self.account - - let uploadedEntrySignal: Signal = unpackedFile - |> mapToSignal { tempFile -> Signal in - return ChatHistoryImport.uploadMedia(account: account, session: session, file: tempFile, fileName: entry.0.path, mimeType: entry.1, type: entry.2) - |> mapError { error -> ImportError in - switch error { - case .chatAdminRequired: - return .chatAdminRequired - case .generic: - return .generic + let entry = self.pendingEntries.removeFirst() + let unpackedFile = Signal { subscriber in + let tempFile = TempBox.shared.tempFile(fileName: entry.0.path) + Logger.shared.log("ChatImportScreen", "Extracting \(entry.0.path) to \(tempFile.path)...") + let startTime = CACurrentMediaTime() + if SSZipArchive.extractFileFromArchive(atPath: archivePath, filePath: entry.0.path, toPath: tempFile.path) { + Logger.shared.log("ChatImportScreen", "[Done in \(CACurrentMediaTime() - startTime) s] Extract \(entry.0.path) to \(tempFile.path)") + subscriber.putNext(tempFile) + subscriber.putCompletion() + } else { + subscriber.putError(.generic) + } + + return EmptyDisposable + } + + let account = self.account + + let uploadedEntrySignal: Signal = unpackedFile + |> mapToSignal { tempFile -> Signal in + return ChatHistoryImport.uploadMedia(account: account, session: session, file: tempFile, fileName: entry.0.path, mimeType: entry.1, type: entry.2) + |> mapError { error -> ImportError in + switch error { + case .chatAdminRequired: + return .chatAdminRequired + case .generic: + return .generic + } } } + + let disposable = MetaDisposable() + self.activeEntries[entry.1] = disposable + + disposable.set((uploadedEntrySignal + |> deliverOnMainQueue).start(next: { [weak self] progress in + guard let strongSelf = self else { + return + } + if let (size, _) = strongSelf.entryProgress[entry.0.path] { + strongSelf.entryProgress[entry.0.path] = (size, Int(progress * Float(entry.0.uncompressedSize))) + strongSelf.updateProgress() + } + }, error: { [weak self] error in + guard let strongSelf = self else { + return + } + strongSelf.failWithError(error) + }, completed: { [weak self] in + guard let strongSelf = self else { + return + } + strongSelf.activeEntries.removeValue(forKey: entry.0.path) + strongSelf.updateState() + })) } - - self.activeEntries[entry.1] = (uploadedEntrySignal - |> deliverOnMainQueue).start(next: { [weak self] progress in - guard let strongSelf = self else { - return - } - if let (size, _) = strongSelf.entryProgress[entry.0.path] { - strongSelf.entryProgress[entry.0.path] = (size, Int(progress * Float(entry.0.uncompressedSize))) - strongSelf.updateProgress() - } - }, error: { [weak self] error in - guard let strongSelf = self else { - return - } - strongSelf.failWithError(error) - }, completed: { [weak self] in - guard let strongSelf = self else { - return - } - strongSelf.activeEntries.removeValue(forKey: entry.0.path) - strongSelf.updateState() - }) } } @@ -652,7 +663,7 @@ public final class ChatImportActivityScreen: ViewController { private var presentationData: PresentationData fileprivate let cancel: () -> Void fileprivate var peerId: PeerId - private let archivePath: String + private let archivePath: String? private let mainEntry: TempBoxFile private let totalBytes: Int private let totalMediaBytes: Int @@ -673,7 +684,7 @@ public final class ChatImportActivityScreen: ViewController { } } - public init(context: AccountContext, cancel: @escaping () -> Void, peerId: PeerId, archivePath: String, mainEntry: TempBoxFile, otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)]) { + public init(context: AccountContext, cancel: @escaping () -> Void, peerId: PeerId, archivePath: String?, mainEntry: TempBoxFile, otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)]) { self.context = context self.cancel = cancel self.peerId = peerId @@ -769,10 +780,8 @@ public final class ChatImportActivityScreen: ViewController { } strongSelf.controllerNode.updateState(state: state, animated: true) if case let .progress(_, _, totalMediaBytes, totalUploadedMediaBytes) = state { - if let progressEstimator = strongSelf.progressEstimator { - let progress = Float(totalUploadedMediaBytes) / Float(totalMediaBytes) - strongSelf.totalMediaProgress = progress - } + let progress = Float(totalUploadedMediaBytes) / Float(totalMediaBytes) + strongSelf.totalMediaProgress = progress } })) }, error: { [weak self] error in diff --git a/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTGzip.h b/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTGzip.h index 3c255592ba..fe16814ea6 100644 --- a/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTGzip.h +++ b/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTGzip.h @@ -5,6 +5,7 @@ NS_ASSUME_NONNULL_BEGIN @interface MTGzip : NSObject + (NSData * _Nullable)decompress:(NSData *)data; ++ (NSData * _Nullable)compress:(NSData *)data; @end diff --git a/submodules/MtProtoKit/Sources/MTGzip.m b/submodules/MtProtoKit/Sources/MTGzip.m index fa7f1ba182..c529dbad29 100644 --- a/submodules/MtProtoKit/Sources/MTGzip.m +++ b/submodules/MtProtoKit/Sources/MTGzip.m @@ -50,4 +50,42 @@ return (retCode == Z_STREAM_END ? result : nil); } ++ (NSData * _Nullable)compress:(NSData *)data { + if (data.length == 0) { + return data; + } + + z_stream stream; + stream.zalloc = Z_NULL; + stream.zfree = Z_NULL; + stream.opaque = Z_NULL; + stream.avail_in = (uint)data.length; + stream.next_in = (Bytef *)(void *)data.bytes; + stream.total_out = 0; + stream.avail_out = 0; + + static const NSUInteger ChunkSize = 16384; + + NSMutableData *output = nil; + int compression = Z_BEST_COMPRESSION; + if (deflateInit2(&stream, compression, Z_DEFLATED, 31, 8, Z_DEFAULT_STRATEGY) == Z_OK) + { + output = [NSMutableData dataWithLength:ChunkSize]; + while (stream.avail_out == 0) + { + if (stream.total_out >= output.length) + { + output.length += ChunkSize; + } + stream.next_out = (uint8_t *)output.mutableBytes + stream.total_out; + stream.avail_out = (uInt)(output.length - stream.total_out); + deflate(&stream, Z_FINISH); + } + deflateEnd(&stream); + output.length = stream.total_out; + } + + return output; +} + @end diff --git a/submodules/TelegramCore/Sources/ChatHistoryImport.swift b/submodules/TelegramCore/Sources/ChatHistoryImport.swift index e202b53bc6..971e63ca83 100644 --- a/submodules/TelegramCore/Sources/ChatHistoryImport.swift +++ b/submodules/TelegramCore/Sources/ChatHistoryImport.swift @@ -49,7 +49,7 @@ public enum ChatHistoryImport { } public static func initSession(account: Account, peerId: PeerId, file: TempBoxFile, mediaCount: Int32) -> Signal { - return multipartUpload(network: account.network, postbox: account.postbox, source: .tempFile(file), encrypt: false, tag: nil, hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) + return multipartUpload(network: account.network, postbox: account.postbox, source: .tempFile(file), encrypt: false, tag: nil, hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false, useLargerParts: true, increaseParallelParts: true, useMultiplexedRequests: false, useCompression: true) |> mapError { _ -> InitImportError in return .generic } diff --git a/submodules/TelegramCore/Sources/Download.swift b/submodules/TelegramCore/Sources/Download.swift index 00d4108fd2..4220b040c9 100644 --- a/submodules/TelegramCore/Sources/Download.swift +++ b/submodules/TelegramCore/Sources/Download.swift @@ -22,6 +22,20 @@ enum UploadPartError { case invalidMedia } +private func wrapMethodBody(_ body: (FunctionDescription, Buffer, DeserializeFunctionResponse), useCompression: Bool) -> (FunctionDescription, Buffer, DeserializeFunctionResponse) { + if useCompression { + if let compressed = MTGzip.compress(body.1.makeData()) { + if compressed.count < body.1.size { + let os = MTOutputStream() + os.write(0x3072cfa1 as Int32) + os.writeBytes(compressed) + return (body.0, Buffer(data: os.currentBytes()), body.2) + } + } + } + + return body +} class Download: NSObject, MTRequestMessageServiceDelegate { let datacenterId: Int @@ -82,7 +96,7 @@ class Download: NSObject, MTRequestMessageServiceDelegate { self.context.authTokenForDatacenter(withIdRequired: self.datacenterId, authToken:self.mtProto.requiredAuthToken, masterDatacenterId: self.mtProto.authTokenMasterDatacenterId) } - static func uploadPart(multiplexedManager: MultiplexedRequestManager, datacenterId: Int, consumerId: Int64, tag: MediaResourceFetchTag?, fileId: Int64, index: Int, data: Data, asBigPart: Bool, bigTotalParts: Int? = nil) -> Signal { + static func uploadPart(multiplexedManager: MultiplexedRequestManager, datacenterId: Int, consumerId: Int64, tag: MediaResourceFetchTag?, fileId: Int64, index: Int, data: Data, asBigPart: Bool, bigTotalParts: Int? = nil, useCompression: Bool = false) -> Signal { let saveFilePart: (FunctionDescription, Buffer, DeserializeFunctionResponse) if asBigPart { let totalParts: Int32 @@ -96,7 +110,7 @@ class Download: NSObject, MTRequestMessageServiceDelegate { saveFilePart = Api.functions.upload.saveFilePart(fileId: fileId, filePart: Int32(index), bytes: Buffer(data: data)) } - return multiplexedManager.request(to: .main(datacenterId), consumerId: consumerId, data: saveFilePart, tag: tag, continueInBackground: true) + return multiplexedManager.request(to: .main(datacenterId), consumerId: consumerId, data: wrapMethodBody(saveFilePart, useCompression: useCompression), tag: tag, continueInBackground: true) |> mapError { error -> UploadPartError in if error.errorCode == 400 { return .invalidMedia @@ -109,11 +123,11 @@ class Download: NSObject, MTRequestMessageServiceDelegate { } } - func uploadPart(fileId: Int64, index: Int, data: Data, asBigPart: Bool, bigTotalParts: Int? = nil) -> Signal { + func uploadPart(fileId: Int64, index: Int, data: Data, asBigPart: Bool, bigTotalParts: Int? = nil, useCompression: Bool = false) -> Signal { return Signal { subscriber in let request = MTRequest() - let saveFilePart: (FunctionDescription, Buffer, DeserializeFunctionResponse) + var saveFilePart: (FunctionDescription, Buffer, DeserializeFunctionResponse) if asBigPart { let totalParts: Int32 if let bigTotalParts = bigTotalParts { @@ -126,6 +140,8 @@ class Download: NSObject, MTRequestMessageServiceDelegate { saveFilePart = Api.functions.upload.saveFilePart(fileId: fileId, filePart: Int32(index), bytes: Buffer(data: data)) } + saveFilePart = wrapMethodBody(saveFilePart, useCompression: useCompression) + request.setPayload(saveFilePart.1.makeData() as Data, metadata: WrappedRequestMetadata(metadata: WrappedFunctionDescription(saveFilePart.0), tag: nil), shortMetadata: WrappedRequestShortMetadata(shortMetadata: WrappedShortFunctionDescription(saveFilePart.0)), responseParser: { response in if let result = saveFilePart.2.parse(Buffer(data: response)) { return BoxedMessage(result) diff --git a/submodules/TelegramCore/Sources/MultipartUpload.swift b/submodules/TelegramCore/Sources/MultipartUpload.swift index 1c094691ac..0736fd4940 100644 --- a/submodules/TelegramCore/Sources/MultipartUpload.swift +++ b/submodules/TelegramCore/Sources/MultipartUpload.swift @@ -113,7 +113,7 @@ private enum HeaderPartState { } private final class MultipartUploadManager { - let parallelParts: Int = 3 + let parallelParts: Int var defaultPartSize: Int var bigTotalParts: Int? var bigParts: Bool @@ -140,13 +140,19 @@ private final class MultipartUploadManager { let state: MultipartUploadState - init(headerSize: Int32, data: Signal, encryptionKey: SecretFileEncryptionKey?, hintFileSize: Int?, hintFileIsLarge: Bool, forceNoBigParts: Bool, useLargerParts: Bool, uploadPart: @escaping (UploadPart) -> Signal, progress: @escaping (Float) -> Void, completed: @escaping (MultipartIntermediateResult?) -> Void) { + init(headerSize: Int32, data: Signal, encryptionKey: SecretFileEncryptionKey?, hintFileSize: Int?, hintFileIsLarge: Bool, forceNoBigParts: Bool, useLargerParts: Bool, increaseParallelParts: Bool, uploadPart: @escaping (UploadPart) -> Signal, progress: @escaping (Float) -> Void, completed: @escaping (MultipartIntermediateResult?) -> Void) { self.dataSignal = data var fileId: Int64 = 0 arc4random_buf(&fileId, 8) self.fileId = fileId + if increaseParallelParts { + self.parallelParts = 30 + } else { + self.parallelParts = 3 + } + self.forceNoBigParts = forceNoBigParts self.useLargerParts = useLargerParts @@ -381,7 +387,7 @@ enum MultipartUploadError { case generic } -func multipartUpload(network: Network, postbox: Postbox, source: MultipartUploadSource, encrypt: Bool, tag: MediaResourceFetchTag?, hintFileSize: Int?, hintFileIsLarge: Bool, forceNoBigParts: Bool, useLargerParts: Bool = false, useMultiplexedRequests: Bool = false) -> Signal { +func multipartUpload(network: Network, postbox: Postbox, source: MultipartUploadSource, encrypt: Bool, tag: MediaResourceFetchTag?, hintFileSize: Int?, hintFileIsLarge: Bool, forceNoBigParts: Bool, useLargerParts: Bool = false, increaseParallelParts: Bool = false, useMultiplexedRequests: Bool = false, useCompression: Bool = false) -> Signal { enum UploadInterface { case download(Download) case multiplexed(manager: MultiplexedRequestManager, datacenterId: Int, consumerId: Int64) @@ -447,12 +453,12 @@ func multipartUpload(network: Network, postbox: Postbox, source: MultipartUpload fetchedResource = .complete() } - let manager = MultipartUploadManager(headerSize: headerSize, data: dataSignal, encryptionKey: encryptionKey, hintFileSize: hintFileSize, hintFileIsLarge: hintFileIsLarge, forceNoBigParts: forceNoBigParts, useLargerParts: useLargerParts, uploadPart: { part in + let manager = MultipartUploadManager(headerSize: headerSize, data: dataSignal, encryptionKey: encryptionKey, hintFileSize: hintFileSize, hintFileIsLarge: hintFileIsLarge, forceNoBigParts: forceNoBigParts, useLargerParts: useLargerParts, increaseParallelParts: increaseParallelParts, uploadPart: { part in switch uploadInterface { case let .download(download): - return download.uploadPart(fileId: part.fileId, index: part.index, data: part.data, asBigPart: part.bigPart, bigTotalParts: part.bigTotalParts) + return download.uploadPart(fileId: part.fileId, index: part.index, data: part.data, asBigPart: part.bigPart, bigTotalParts: part.bigTotalParts, useCompression: useCompression) case let .multiplexed(multiplexed, datacenterId, consumerId): - return Download.uploadPart(multiplexedManager: multiplexed, datacenterId: datacenterId, consumerId: consumerId, tag: nil, fileId: part.fileId, index: part.index, data: part.data, asBigPart: part.bigPart, bigTotalParts: part.bigTotalParts) + return Download.uploadPart(multiplexedManager: multiplexed, datacenterId: datacenterId, consumerId: consumerId, tag: nil, fileId: part.fileId, index: part.index, data: part.data, asBigPart: part.bigPart, bigTotalParts: part.bigTotalParts, useCompression: useCompression) } }, progress: { progress in subscriber.putNext(.progress(progress)) diff --git a/submodules/TelegramUI/Sources/ShareExtensionContext.swift b/submodules/TelegramUI/Sources/ShareExtensionContext.swift index 8b43cca864..4846204af6 100644 --- a/submodules/TelegramUI/Sources/ShareExtensionContext.swift +++ b/submodules/TelegramUI/Sources/ShareExtensionContext.swift @@ -401,81 +401,98 @@ public class ShareRootControllerImpl { return } let fileExtension = (fileName as NSString).pathExtension - guard fileExtension.lowercased() == "zip" else { - beginShare() - return - } - - let archivePath = url.path - - guard let entries = SSZipArchive.getEntriesForFile(atPath: archivePath) else { - beginShare() - return - } - - let mainFileNames: [NSRegularExpression] = [ - try! NSRegularExpression(pattern: "_chat\\.txt"), - try! NSRegularExpression(pattern: "KakaoTalkChats\\.txt"), - try! NSRegularExpression(pattern: "Talk_.*?\\.txt"), - ] - - var maybeMainFileName: String? - mainFileLoop: for entry in entries { - let entryFileName = entry.path.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "..", with: "_") - let fullRange = NSRange(entryFileName.startIndex ..< entryFileName.endIndex, in: entryFileName) - for expression in mainFileNames { - if expression.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { - maybeMainFileName = entryFileName - break mainFileLoop - } - } - } - - guard let mainFileName = maybeMainFileName else { - beginShare() - return - } - - let photoRegex = try! NSRegularExpression(pattern: ".*?\\.jpg") - let videoRegex = try! NSRegularExpression(pattern: "[\\d]+-VIDEO-.*?\\.mp4") - let stickerRegex = try! NSRegularExpression(pattern: "[\\d]+-STICKER-.*?\\.webp") - let voiceRegex = try! NSRegularExpression(pattern: "[\\d]+-AUDIO-.*?\\.opus") + var archivePathValue: String? var otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] = [] - var mainFile: TempBoxFile? - do { - for entry in entries { - let entryPath = entry.path.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "..", with: "_") - if entryPath.isEmpty { - continue - } - let tempFile = TempBox.shared.tempFile(fileName: entryPath) - if entryPath == mainFileName { - if SSZipArchive.extractFileFromArchive(atPath: archivePath, filePath: entry.path, toPath: tempFile.path) { - mainFile = tempFile - } - } else { - let entryFileName = (entryPath as NSString).lastPathComponent - if !entryFileName.isEmpty { - let mediaType: ChatHistoryImport.MediaType - let fullRange = NSRange(entryFileName.startIndex ..< entryFileName.endIndex, in: entryFileName) - if photoRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { - mediaType = .photo - } else if videoRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { - mediaType = .video - } else if stickerRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { - mediaType = .sticker - } else if voiceRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { - mediaType = .voice - } else { - mediaType = .file - } - otherEntries.append((entry, entryFileName, mediaType)) + + if fileExtension.lowercased() == "zip" { + let archivePath = url.path + archivePathValue = archivePath + + guard let entries = SSZipArchive.getEntriesForFile(atPath: archivePath) else { + beginShare() + return + } + + let mainFileNames: [NSRegularExpression] = [ + try! NSRegularExpression(pattern: "_chat\\.txt"), + try! NSRegularExpression(pattern: "KakaoTalkChats\\.txt"), + try! NSRegularExpression(pattern: "Talk_.*?\\.txt"), + ] + + var maybeMainFileName: String? + mainFileLoop: for entry in entries { + let entryFileName = entry.path.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "..", with: "_") + let fullRange = NSRange(entryFileName.startIndex ..< entryFileName.endIndex, in: entryFileName) + for expression in mainFileNames { + if expression.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { + maybeMainFileName = entryFileName + break mainFileLoop } } } - } catch { + + guard let mainFileName = maybeMainFileName else { + beginShare() + return + } + + let photoRegex = try! NSRegularExpression(pattern: ".*?\\.jpg") + let videoRegex = try! NSRegularExpression(pattern: "[\\d]+-VIDEO-.*?\\.mp4") + let stickerRegex = try! NSRegularExpression(pattern: "[\\d]+-STICKER-.*?\\.webp") + let voiceRegex = try! NSRegularExpression(pattern: "[\\d]+-AUDIO-.*?\\.opus") + + do { + for entry in entries { + let entryPath = entry.path.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "..", with: "_") + if entryPath.isEmpty { + continue + } + let tempFile = TempBox.shared.tempFile(fileName: entryPath) + if entryPath == mainFileName { + if SSZipArchive.extractFileFromArchive(atPath: archivePath, filePath: entry.path, toPath: tempFile.path) { + mainFile = tempFile + } + } else { + let entryFileName = (entryPath as NSString).lastPathComponent + if !entryFileName.isEmpty { + let mediaType: ChatHistoryImport.MediaType + let fullRange = NSRange(entryFileName.startIndex ..< entryFileName.endIndex, in: entryFileName) + if photoRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { + mediaType = .photo + } else if videoRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { + mediaType = .video + } else if stickerRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { + mediaType = .sticker + } else if voiceRegex.firstMatch(in: entryFileName, options: [], range: fullRange) != nil { + mediaType = .voice + } else { + mediaType = .file + } + otherEntries.append((entry, entryFileName, mediaType)) + } + } + } + } + } else if fileExtension.lowercased() == "txt" { + if let mainFileText = try? String(contentsOf: URL(fileURLWithPath: url.path)) { + if !mainFileText.hasPrefix("[LINE]") { + beginShare() + return + } + } else { + beginShare() + return + } + + let tempFile = TempBox.shared.tempFile(fileName: "History.txt") + if let _ = try? FileManager.default.copyItem(atPath: url.path, toPath: tempFile.path) { + mainFile = tempFile + } else { + beginShare() + return + } } if let mainFile = mainFile, let mainFileText = try? String(contentsOf: URL(fileURLWithPath: mainFile.path)) { @@ -561,7 +578,7 @@ public class ShareRootControllerImpl { navigationController.view.endEditing(true) navigationController.pushViewController(ChatImportActivityScreen(context: context, cancel: { self?.getExtensionContext()?.completeRequest(returningItems: nil, completionHandler: nil) - }, peerId: peerId, archivePath: archivePath, mainEntry: mainFile, otherEntries: otherEntries)) + }, peerId: peerId, archivePath: archivePathValue, mainEntry: mainFile, otherEntries: otherEntries)) } attemptSelectionImpl = { peer in @@ -675,7 +692,7 @@ public class ShareRootControllerImpl { navigationController.view.endEditing(true) navigationController.pushViewController(ChatImportActivityScreen(context: context, cancel: { self?.getExtensionContext()?.completeRequest(returningItems: nil, completionHandler: nil) - }, peerId: peerId, archivePath: archivePath, mainEntry: mainFile, otherEntries: otherEntries)) + }, peerId: peerId, archivePath: archivePathValue, mainEntry: mainFile, otherEntries: otherEntries)) } attemptSelectionImpl = { [weak controller] peer in @@ -737,7 +754,7 @@ public class ShareRootControllerImpl { navigationController.view.endEditing(true) navigationController.pushViewController(ChatImportActivityScreen(context: context, cancel: { self?.getExtensionContext()?.completeRequest(returningItems: nil, completionHandler: nil) - }, peerId: peerId, archivePath: archivePath, mainEntry: mainFile, otherEntries: otherEntries)) + }, peerId: peerId, archivePath: archivePathValue, mainEntry: mainFile, otherEntries: otherEntries)) } attemptSelectionImpl = { [weak controller] peer in From 0fbb84963755d75e16b872621dc5085fe40b2343 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 19:49:00 +0500 Subject: [PATCH 5/8] Fix video rotation --- submodules/MediaPlayer/Sources/MediaPlayerNode.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/MediaPlayer/Sources/MediaPlayerNode.swift b/submodules/MediaPlayer/Sources/MediaPlayerNode.swift index 13c6985ff0..d8abad3a3f 100644 --- a/submodules/MediaPlayer/Sources/MediaPlayerNode.swift +++ b/submodules/MediaPlayer/Sources/MediaPlayerNode.swift @@ -103,7 +103,7 @@ public final class MediaPlayerNode: ASDisplayNode { self.currentRotationAngle = rotationAngle self.currentAspect = aspect var transform = CGAffineTransform(rotationAngle: CGFloat(rotationAngle)) - if !rotationAngle.isZero { + if abs(rotationAngle).remainder(dividingBy: Double.pi / 2.0) > 0.1 { transform = transform.scaledBy(x: CGFloat(aspect), y: CGFloat(1.0 / aspect)) } videoLayer.setAffineTransform(transform) From 145c034ad85aaff39f263c991de9bc01e2e95e33 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 20:50:58 +0500 Subject: [PATCH 6/8] Fix video rotation --- submodules/MediaPlayer/Sources/MediaPlayerNode.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/MediaPlayer/Sources/MediaPlayerNode.swift b/submodules/MediaPlayer/Sources/MediaPlayerNode.swift index d8abad3a3f..d56c56aee9 100644 --- a/submodules/MediaPlayer/Sources/MediaPlayerNode.swift +++ b/submodules/MediaPlayer/Sources/MediaPlayerNode.swift @@ -103,7 +103,7 @@ public final class MediaPlayerNode: ASDisplayNode { self.currentRotationAngle = rotationAngle self.currentAspect = aspect var transform = CGAffineTransform(rotationAngle: CGFloat(rotationAngle)) - if abs(rotationAngle).remainder(dividingBy: Double.pi / 2.0) > 0.1 { + if abs(rotationAngle).remainder(dividingBy: Double.pi) > 0.1 { transform = transform.scaledBy(x: CGFloat(aspect), y: CGFloat(1.0 / aspect)) } videoLayer.setAffineTransform(transform) From ee1f85226b64821adf91e88b1aaebee003535d87 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 22:09:33 +0500 Subject: [PATCH 7/8] Fix chat import layout --- .../Telegram-iOS/en.lproj/Localizable.strings | 2 +- .../Sources/ChatImportActivityScreen.swift | 48 +++++++++++++++---- .../Sources/ChatHistoryImport.swift | 4 +- .../Sources/ShareExtensionContext.swift | 2 +- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/Telegram/Telegram-iOS/en.lproj/Localizable.strings b/Telegram/Telegram-iOS/en.lproj/Localizable.strings index 32be79e0c4..9c69517469 100644 --- a/Telegram/Telegram-iOS/en.lproj/Localizable.strings +++ b/Telegram/Telegram-iOS/en.lproj/Localizable.strings @@ -5957,7 +5957,7 @@ Sorry for the inconvenience."; "ChatImportActivity.InProgress" = "Please keep this window open\nduring the import."; "ChatImportActivity.ErrorNotAdmin" = "You need to be an admin."; "ChatImportActivity.ErrorInvalidChatType" = "You can't import this history in this type of chat."; -"ChatImportActivity.ErrorUserBlocked" = "You need to be an admin."; +"ChatImportActivity.ErrorUserBlocked" = "This user is blocked."; "ChatImportActivity.ErrorGeneric" = "An error occurred."; "ChatImportActivity.Success" = "Chat imported\nsuccessfully."; diff --git a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift index eead72c488..a5101d6516 100644 --- a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift +++ b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift @@ -56,6 +56,7 @@ private final class ImportManager { case generic case chatAdminRequired case invalidChatType + case userBlocked } enum State { @@ -116,6 +117,8 @@ private final class ImportManager { return .invalidChatType case .generic: return .generic + case .userBlocked: + return .userBlocked } } |> deliverOnMainQueue).start(next: { [weak self] session in @@ -449,12 +452,39 @@ public final class ChatImportActivityScreen: ViewController { let isFirstLayout = self.validLayout == nil self.validLayout = (layout, navigationHeight) - let iconSize = CGSize(width: 190.0, height: 190.0) - let radialStatusSize = CGSize(width: 186.0, height: 186.0) - let maxIconStatusSpacing: CGFloat = 46.0 - let maxProgressTextSpacing: CGFloat = 33.0 - let progressStatusSpacing: CGFloat = 14.0 - let statusButtonSpacing: CGFloat = 19.0 + let availableHeight = layout.size.height - navigationHeight + + var iconSize = CGSize(width: 190.0, height: 190.0) + var radialStatusSize = CGSize(width: 186.0, height: 186.0) + var maxIconStatusSpacing: CGFloat = 46.0 + var maxProgressTextSpacing: CGFloat = 33.0 + var progressStatusSpacing: CGFloat = 14.0 + var statusButtonSpacing: CGFloat = 19.0 + + var maxK: CGFloat = availableHeight / (iconSize.height + maxIconStatusSpacing + 30.0 + maxProgressTextSpacing + 320.0) + maxK = max(0.5, min(1.0, maxK)) + + iconSize.width = floor(iconSize.width * maxK) + iconSize.height = floor(iconSize.height * maxK) + radialStatusSize.width = floor(radialStatusSize.width * maxK) + radialStatusSize.height = floor(radialStatusSize.height * maxK) + maxIconStatusSpacing = floor(maxIconStatusSpacing * maxK) + maxProgressTextSpacing = floor(maxProgressTextSpacing * maxK) + progressStatusSpacing = floor(progressStatusSpacing * maxK) + statusButtonSpacing = floor(statusButtonSpacing * maxK) + + var updateRadialBackround = false + if let width = self.radialStatusBackground.image?.size.width { + if abs(width - radialStatusSize.width) > 0.01 { + updateRadialBackround = true + } + } else { + updateRadialBackround = true + } + + if updateRadialBackround { + self.radialStatusBackground.image = generateCircleImage(diameter: radialStatusSize.width, lineWidth: 6.0, color: self.presentationData.theme.list.itemAccentColor.withMultipliedAlpha(0.2)) + } let effectiveProgress: CGFloat switch state { @@ -470,7 +500,7 @@ public final class ChatImportActivityScreen: ViewController { effectiveProgress = 1.0 } - self.radialStatusText.attributedText = NSAttributedString(string: "\(Int(effectiveProgress * 100.0))%", font: Font.with(size: 36.0, design: .round, weight: .semibold), textColor: self.presentationData.theme.list.itemPrimaryTextColor) + self.radialStatusText.attributedText = NSAttributedString(string: "\(Int(effectiveProgress * 100.0))%", font: Font.with(size: floor(36.0 * maxK), design: .round, weight: .semibold), textColor: self.presentationData.theme.list.itemPrimaryTextColor) let radialStatusTextSize = self.radialStatusText.updateLayout(CGSize(width: 200.0, height: .greatestFiniteMagnitude)) self.progressText.attributedText = NSAttributedString(string: "\(dataSizeString(Int(effectiveProgress * CGFloat(self.totalBytes)))) of \(dataSizeString(Int(1.0 * CGFloat(self.totalBytes))))", font: Font.semibold(17.0), textColor: self.presentationData.theme.list.itemPrimaryTextColor) @@ -496,6 +526,8 @@ public final class ChatImportActivityScreen: ViewController { errorText = self.presentationData.strings.ChatImportActivity_ErrorInvalidChatType case .generic: errorText = self.presentationData.strings.ChatImportActivity_ErrorGeneric + case .userBlocked: + errorText = self.presentationData.strings.ChatImportActivity_ErrorUserBlocked } self.statusText.attributedText = NSAttributedString(string: errorText, font: Font.regular(17.0), textColor: self.presentationData.theme.list.itemDestructiveColor) case .done: @@ -510,7 +542,7 @@ public final class ChatImportActivityScreen: ViewController { hideIcon = true contentHeight = progressTextSize.height + progressStatusSpacing + 160.0 } else { - contentHeight = iconSize.height + maxIconStatusSpacing + radialStatusSize.height + maxProgressTextSpacing + progressTextSize.height + progressStatusSpacing + 100.0 + contentHeight = iconSize.height + maxIconStatusSpacing + radialStatusSize.height + maxProgressTextSpacing + progressTextSize.height + progressStatusSpacing + 140.0 } transition.updateAlpha(node: self.radialStatus, alpha: hideIcon ? 0.0 : 1.0) diff --git a/submodules/TelegramCore/Sources/ChatHistoryImport.swift b/submodules/TelegramCore/Sources/ChatHistoryImport.swift index 971e63ca83..48f455b609 100644 --- a/submodules/TelegramCore/Sources/ChatHistoryImport.swift +++ b/submodules/TelegramCore/Sources/ChatHistoryImport.swift @@ -2,7 +2,6 @@ import Foundation import SwiftSignalKit import Postbox import SyncCore -import TelegramCore import TelegramApi public enum ChatHistoryImport { @@ -16,6 +15,7 @@ public enum ChatHistoryImport { case generic case chatAdminRequired case invalidChatType + case userBlocked } public enum ParsedInfo { @@ -71,6 +71,8 @@ public enum ChatHistoryImport { return .chatAdminRequired case "IMPORT_PEER_TYPE_INVALID": return .invalidChatType + case "USER_IS_BLOCKED": + return .userBlocked default: return .generic } diff --git a/submodules/TelegramUI/Sources/ShareExtensionContext.swift b/submodules/TelegramUI/Sources/ShareExtensionContext.swift index 4846204af6..84b47ebdfe 100644 --- a/submodules/TelegramUI/Sources/ShareExtensionContext.swift +++ b/submodules/TelegramUI/Sources/ShareExtensionContext.swift @@ -542,7 +542,7 @@ public class ShareRootControllerImpl { super.containerLayoutUpdated(layout, transition: transition) let indicatorSize = self.activityIndicator.measure(CGSize(width: 100.0, height: 100.0)) - transition.updateFrame(node: self.activityIndicator, frame: CGRect(origin: CGPoint(x: floor((layout.size.width - indicatorSize.width) / 2.0), y: floor((layout.size.height - indicatorSize.height - 50.0) / 2.0)), size: indicatorSize)) + transition.updateFrame(node: self.activityIndicator, frame: CGRect(origin: CGPoint(x: floor((layout.size.width - indicatorSize.width) / 2.0), y: self.navigationHeight + floor((layout.size.height - self.navigationHeight - indicatorSize.height) / 2.0)), size: indicatorSize)) } } From ed95c332f3b9af176271978fc739d59c3e7d88c2 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 26 Jan 2021 22:47:29 +0500 Subject: [PATCH 8/8] Accept regular expressions from server --- .../Sources/ShareExtensionContext.swift | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/submodules/TelegramUI/Sources/ShareExtensionContext.swift b/submodules/TelegramUI/Sources/ShareExtensionContext.swift index 84b47ebdfe..f21a6952ae 100644 --- a/submodules/TelegramUI/Sources/ShareExtensionContext.swift +++ b/submodules/TelegramUI/Sources/ShareExtensionContext.swift @@ -406,6 +406,25 @@ public class ShareRootControllerImpl { var otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType)] = [] var mainFile: TempBoxFile? + let appConfiguration = context.currentAppConfiguration.with({ $0 }) + + /* + history_import_filters: { + "zip": { + "main_file_patterns": [ + "_chat\\.txt", + "KakaoTalkChats\\.txt", + "Talk_.*?\\.txt" + ] + }, + "txt": { + "patterns": [ + "^\\[LINE\\]" + ] + } + } + */ + if fileExtension.lowercased() == "zip" { let archivePath = url.path archivePathValue = archivePath @@ -415,12 +434,24 @@ public class ShareRootControllerImpl { return } - let mainFileNames: [NSRegularExpression] = [ - try! NSRegularExpression(pattern: "_chat\\.txt"), - try! NSRegularExpression(pattern: "KakaoTalkChats\\.txt"), - try! NSRegularExpression(pattern: "Talk_.*?\\.txt"), + var mainFileNameExpressions: [String] = [ + "_chat\\.txt", + "KakaoTalkChats\\.txt", + "Talk_.*?\\.txt", ] + if let data = appConfiguration.data, let dict = data["history_import_filters"] as? [String: Any] { + if let zip = dict["zip"] as? [String: Any] { + if let patterns = zip["main_file_patterns"] as? [String] { + mainFileNameExpressions = patterns + } + } + } + + let mainFileNames: [NSRegularExpression] = mainFileNameExpressions.compactMap { string -> NSRegularExpression? in + return try? NSRegularExpression(pattern: string) + } + var maybeMainFileName: String? mainFileLoop: for entry in entries { let entryFileName = entry.path.replacingOccurrences(of: "/", with: "_").replacingOccurrences(of: "..", with: "_") @@ -476,8 +507,32 @@ public class ShareRootControllerImpl { } } } else if fileExtension.lowercased() == "txt" { + var fileScanExpressions: [String] = [ + "^\\[LINE\\]", + ] + + if let data = appConfiguration.data, let dict = data["history_import_filters"] as? [String: Any] { + if let zip = dict["txt"] as? [String: Any] { + if let patterns = zip["patterns"] as? [String] { + fileScanExpressions = patterns + } + } + } + + let filePatterns: [NSRegularExpression] = fileScanExpressions.compactMap { string -> NSRegularExpression? in + return try? NSRegularExpression(pattern: string) + } + if let mainFileText = try? String(contentsOf: URL(fileURLWithPath: url.path)) { - if !mainFileText.hasPrefix("[LINE]") { + let fullRange = NSRange(mainFileText.startIndex ..< mainFileText.endIndex, in: mainFileText) + var foundMatch = false + for pattern in filePatterns { + if pattern.firstMatch(in: mainFileText, options: [], range: fullRange) != nil { + foundMatch = true + break + } + } + if !foundMatch { beginShare() return }