diff --git a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift index 716b0bdf2b..46891b2e4d 100644 --- a/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift +++ b/submodules/ChatImportUI/Sources/ChatImportActivityScreen.swift @@ -454,7 +454,7 @@ public final class ChatImportActivityScreen: ViewController { private let archivePath: String private let mainEntry: TempBoxFile private let mainEntrySize: Int - private let otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType, Promise)] + private let otherEntries: [(SSZipEntry, String, ChatHistoryImport.MediaType, Signal)] private let totalBytes: Int private let totalMediaBytes: Int @@ -480,11 +480,14 @@ public final class ChatImportActivityScreen: ViewController { self.archivePath = archivePath self.mainEntry = mainEntry - self.otherEntries = otherEntries.map { entry -> (SSZipEntry, String, ChatHistoryImport.MediaType, Promise) in + var isFirstFile = true + self.otherEntries = otherEntries.map { entry -> (SSZipEntry, String, ChatHistoryImport.MediaType, Signal) in let signal = Signal { subscriber in let tempFile = TempBox.shared.tempFile(fileName: entry.1) + print("Extracting \(entry.0.path) to \(tempFile.path)...") + let startTime = CACurrentMediaTime() if SSZipArchive.extractFileFromArchive(atPath: archivePath, filePath: entry.0.path, toPath: tempFile.path) { - //print("Extract \(entry.0.path) to \(tempFile.path)") + print("[Done in \(CACurrentMediaTime() - startTime) s] Extract \(entry.0.path) to \(tempFile.path)") subscriber.putNext(tempFile) subscriber.putCompletion() } else { @@ -494,10 +497,9 @@ public final class ChatImportActivityScreen: ViewController { return EmptyDisposable } - |> runOn(Queue.concurrentDefaultQueue()) - let promise = Promise() - promise.set(signal) - return (entry.0, entry.1, entry.2, promise) + //let promise = Promise() + //promise.set(signal) + return (entry.0, entry.1, entry.2, signal) } if let size = fileSize(self.mainEntry.path) { @@ -610,7 +612,7 @@ public final class ChatImportActivityScreen: ViewController { var mediaSignals: [Signal<(String, Float), ImportError>] = [] for (_, fileName, mediaType, fileData) in otherEntries { - let unpackedFile: Signal = fileData.get() + let unpackedFile: Signal = fileData |> take(1) |> deliverOnMainQueue |> castError(ImportError.self) diff --git a/submodules/TelegramCore/Sources/ChatHistoryImport.swift b/submodules/TelegramCore/Sources/ChatHistoryImport.swift index 1ca297b9b6..e202b53bc6 100644 --- a/submodules/TelegramCore/Sources/ChatHistoryImport.swift +++ b/submodules/TelegramCore/Sources/ChatHistoryImport.swift @@ -49,7 +49,7 @@ public enum ChatHistoryImport { } public static func initSession(account: Account, peerId: PeerId, file: TempBoxFile, mediaCount: Int32) -> Signal { - return multipartUpload(network: account.network, postbox: account.postbox, source: .tempFile(file), encrypt: false, tag: nil, hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: account.network, postbox: account.postbox, source: .tempFile(file), encrypt: false, tag: nil, hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> InitImportError in return .generic } @@ -104,7 +104,12 @@ public enum ChatHistoryImport { } public static func uploadMedia(account: Account, session: Session, file: TempBoxFile, fileName: String, mimeType: String, type: MediaType) -> Signal { - return multipartUpload(network: account.network, postbox: account.postbox, source: .tempFile(file), encrypt: false, tag: nil, hintFileSize: nil, hintFileIsLarge: false, useLargerParts: true, useMultiplexedRequests: true) + var forceNoBigParts = true + if let size = fileSize(file.path), size >= 30 * 1024 * 1024 { + forceNoBigParts = false + } + + return multipartUpload(network: account.network, postbox: account.postbox, source: .tempFile(file), encrypt: false, tag: nil, hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: forceNoBigParts, useLargerParts: true, useMultiplexedRequests: true) |> mapError { _ -> UploadMediaError in return .generic } diff --git a/submodules/TelegramCore/Sources/MessageMediaPreuploadManager.swift b/submodules/TelegramCore/Sources/MessageMediaPreuploadManager.swift index 28bff2285f..b60f898f27 100644 --- a/submodules/TelegramCore/Sources/MessageMediaPreuploadManager.swift +++ b/submodules/TelegramCore/Sources/MessageMediaPreuploadManager.swift @@ -37,7 +37,7 @@ private final class MessageMediaPreuploadManagerContext { let context = MessageMediaPreuploadManagerUploadContext() self.uploadContexts[id] = context let queue = self.queue - context.disposable.set(multipartUpload(network: network, postbox: postbox, source: .custom(source), encrypt: encrypt, tag: tag, hintFileSize: nil, hintFileIsLarge: false).start(next: { [weak self] next in + context.disposable.set(multipartUpload(network: network, postbox: postbox, source: .custom(source), encrypt: encrypt, tag: tag, hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false).start(next: { [weak self] next in queue.async { if let strongSelf = self, let context = strongSelf.uploadContexts[id] { switch next { @@ -86,7 +86,7 @@ private final class MessageMediaPreuploadManagerContext { } } } else { - return multipartUpload(network: network, postbox: postbox, source: source, encrypt: encrypt, tag: tag, hintFileSize: hintFileSize, hintFileIsLarge: hintFileIsLarge).start(next: { next in + return multipartUpload(network: network, postbox: postbox, source: source, encrypt: encrypt, tag: tag, hintFileSize: hintFileSize, hintFileIsLarge: hintFileIsLarge, forceNoBigParts: false).start(next: { next in subscriber.putNext(next) }, error: { error in subscriber.putError(error) diff --git a/submodules/TelegramCore/Sources/MultipartUpload.swift b/submodules/TelegramCore/Sources/MultipartUpload.swift index 1b788b37ec..1c094691ac 100644 --- a/submodules/TelegramCore/Sources/MultipartUpload.swift +++ b/submodules/TelegramCore/Sources/MultipartUpload.swift @@ -117,6 +117,7 @@ private final class MultipartUploadManager { var defaultPartSize: Int var bigTotalParts: Int? var bigParts: Bool + private let forceNoBigParts: Bool private let useLargerParts: Bool let queue = Queue() @@ -139,13 +140,14 @@ private final class MultipartUploadManager { let state: MultipartUploadState - init(headerSize: Int32, data: Signal, encryptionKey: SecretFileEncryptionKey?, hintFileSize: Int?, hintFileIsLarge: Bool, useLargerParts: Bool, uploadPart: @escaping (UploadPart) -> Signal, progress: @escaping (Float) -> Void, completed: @escaping (MultipartIntermediateResult?) -> Void) { + init(headerSize: Int32, data: Signal, encryptionKey: SecretFileEncryptionKey?, hintFileSize: Int?, hintFileIsLarge: Bool, forceNoBigParts: Bool, useLargerParts: Bool, uploadPart: @escaping (UploadPart) -> Signal, progress: @escaping (Float) -> Void, completed: @escaping (MultipartIntermediateResult?) -> Void) { self.dataSignal = data var fileId: Int64 = 0 arc4random_buf(&fileId, 8) self.fileId = fileId + self.forceNoBigParts = forceNoBigParts self.useLargerParts = useLargerParts self.state = MultipartUploadState(encryptionKey: encryptionKey) @@ -161,11 +163,11 @@ private final class MultipartUploadManager { self.headerPartState = .notStarted } - if let hintFileSize = hintFileSize, hintFileSize > 10 * 1024 * 1024 { + if let hintFileSize = hintFileSize, hintFileSize > 10 * 1024 * 1024, !forceNoBigParts { self.defaultPartSize = 512 * 1024 self.bigTotalParts = (hintFileSize / self.defaultPartSize) + (hintFileSize % self.defaultPartSize == 0 ? 0 : 1) self.bigParts = true - } else if hintFileIsLarge { + } else if hintFileIsLarge, !forceNoBigParts { self.defaultPartSize = 512 * 1024 self.bigTotalParts = nil self.bigParts = true @@ -203,7 +205,7 @@ private final class MultipartUploadManager { func checkState() { if let resourceData = self.resourceData, resourceData.complete && resourceData.size != 0 { if self.committedOffset == 0 && self.uploadedParts.isEmpty && self.uploadingParts.isEmpty { - if resourceData.size > 10 * 1024 * 1024 { + if resourceData.size > 10 * 1024 * 1024, !self.forceNoBigParts { self.defaultPartSize = 512 * 1024 self.bigTotalParts = (resourceData.size / self.defaultPartSize) + (resourceData.size % self.defaultPartSize == 0 ? 0 : 1) self.bigParts = true @@ -379,7 +381,7 @@ enum MultipartUploadError { case generic } -func multipartUpload(network: Network, postbox: Postbox, source: MultipartUploadSource, encrypt: Bool, tag: MediaResourceFetchTag?, hintFileSize: Int?, hintFileIsLarge: Bool, useLargerParts: Bool = false, useMultiplexedRequests: Bool = false) -> Signal { +func multipartUpload(network: Network, postbox: Postbox, source: MultipartUploadSource, encrypt: Bool, tag: MediaResourceFetchTag?, hintFileSize: Int?, hintFileIsLarge: Bool, forceNoBigParts: Bool, useLargerParts: Bool = false, useMultiplexedRequests: Bool = false) -> Signal { enum UploadInterface { case download(Download) case multiplexed(manager: MultiplexedRequestManager, datacenterId: Int, consumerId: Int64) @@ -445,7 +447,7 @@ func multipartUpload(network: Network, postbox: Postbox, source: MultipartUpload fetchedResource = .complete() } - let manager = MultipartUploadManager(headerSize: headerSize, data: dataSignal, encryptionKey: encryptionKey, hintFileSize: hintFileSize, hintFileIsLarge: hintFileIsLarge, useLargerParts: useLargerParts, uploadPart: { part in + let manager = MultipartUploadManager(headerSize: headerSize, data: dataSignal, encryptionKey: encryptionKey, hintFileSize: hintFileSize, hintFileIsLarge: hintFileIsLarge, forceNoBigParts: forceNoBigParts, useLargerParts: useLargerParts, uploadPart: { part in switch uploadInterface { case let .download(download): return download.uploadPart(fileId: part.fileId, index: part.index, data: part.data, asBigPart: part.bigPart, bigTotalParts: part.bigTotalParts) diff --git a/submodules/TelegramCore/Sources/PeerPhotoUpdater.swift b/submodules/TelegramCore/Sources/PeerPhotoUpdater.swift index b7ecbc0047..32325fe91c 100644 --- a/submodules/TelegramCore/Sources/PeerPhotoUpdater.swift +++ b/submodules/TelegramCore/Sources/PeerPhotoUpdater.swift @@ -38,7 +38,7 @@ enum UploadedPeerPhotoDataContent { } public func uploadedPeerPhoto(postbox: Postbox, network: Network, resource: MediaResource) -> Signal { - return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> map { result -> UploadedPeerPhotoData in return UploadedPeerPhotoData(resource: resource, content: .result(result)) } @@ -57,7 +57,7 @@ public func uploadedPeerVideo(postbox: Postbox, network: Network, messageMediaPr return .single(UploadedPeerPhotoData(resource: resource, content: .error)) } } else { - return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .video), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .video), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> map { result -> UploadedPeerPhotoData in return UploadedPeerPhotoData(resource: resource, content: .result(result)) } diff --git a/submodules/TelegramCore/Sources/PendingMessageUploadedContent.swift b/submodules/TelegramCore/Sources/PendingMessageUploadedContent.swift index d675ac5208..ee975fd5b5 100644 --- a/submodules/TelegramCore/Sources/PendingMessageUploadedContent.swift +++ b/submodules/TelegramCore/Sources/PendingMessageUploadedContent.swift @@ -406,7 +406,7 @@ private func uploadedMediaImageContent(network: Network, postbox: Postbox, trans } else { imageReference = .standalone(media: transformedImage) } - return multipartUpload(network: network, postbox: postbox, source: .resource(imageReference.resourceReference(largestRepresentation.resource)), encrypt: peerId.namespace == Namespaces.Peer.SecretChat, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .resource(imageReference.resourceReference(largestRepresentation.resource)), encrypt: peerId.namespace == Namespaces.Peer.SecretChat, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> PendingMessageUploadError in return .generic } |> mapToSignal { next -> Signal in switch next { @@ -560,7 +560,7 @@ private enum UploadedMediaFileAndThumbnail { } private func uploadedThumbnail(network: Network, postbox: Postbox, resourceReference: MediaResourceReference) -> Signal { - return multipartUpload(network: network, postbox: postbox, source: .resource(resourceReference), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .resource(resourceReference), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> PendingMessageUploadError in return .generic } |> mapToSignal { result -> Signal in switch result { diff --git a/submodules/TelegramCore/Sources/StandaloneSendMessage.swift b/submodules/TelegramCore/Sources/StandaloneSendMessage.swift index 8a3b1aaa63..f43e7c9b91 100644 --- a/submodules/TelegramCore/Sources/StandaloneSendMessage.swift +++ b/submodules/TelegramCore/Sources/StandaloneSendMessage.swift @@ -144,7 +144,7 @@ private enum UploadMediaEvent { } private func uploadedImage(account: Account, data: Data) -> Signal { - return multipartUpload(network: account.network, postbox: account.postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: account.network, postbox: account.postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> StandaloneSendMessageError in return .generic } |> map { next -> UploadMediaEvent in switch next { @@ -159,7 +159,7 @@ private func uploadedImage(account: Account, data: Data) -> Signal Signal { - return multipartUpload(network: account.network, postbox: account.postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: statsCategoryForFileWithAttributes(attributes)), hintFileSize: data.count, hintFileIsLarge: false) + return multipartUpload(network: account.network, postbox: account.postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: statsCategoryForFileWithAttributes(attributes)), hintFileSize: data.count, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> PendingMessageUploadError in return .generic } |> map { next -> UploadMediaEvent in switch next { diff --git a/submodules/TelegramCore/Sources/StandaloneUploadedMedia.swift b/submodules/TelegramCore/Sources/StandaloneUploadedMedia.swift index 90b5062a61..a704f06335 100644 --- a/submodules/TelegramCore/Sources/StandaloneUploadedMedia.swift +++ b/submodules/TelegramCore/Sources/StandaloneUploadedMedia.swift @@ -39,7 +39,7 @@ public enum StandaloneUploadMediaEvent { } private func uploadedThumbnail(network: Network, postbox: Postbox, data: Data) -> Signal { - return multipartUpload(network: network, postbox: postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> StandaloneUploadMediaError in return .generic } |> mapToSignal { result -> Signal in switch result { @@ -54,7 +54,7 @@ private func uploadedThumbnail(network: Network, postbox: Postbox, data: Data) - } public func standaloneUploadedImage(account: Account, peerId: PeerId, text: String, data: Data, thumbnailData: Data? = nil, dimensions: PixelDimensions) -> Signal { - return multipartUpload(network: account.network, postbox: account.postbox, source: .data(data), encrypt: peerId.namespace == Namespaces.Peer.SecretChat, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: account.network, postbox: account.postbox, source: .data(data), encrypt: peerId.namespace == Namespaces.Peer.SecretChat, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> StandaloneUploadMediaError in return .generic } |> mapToSignal { next -> Signal in switch next { @@ -115,7 +115,7 @@ public func standaloneUploadedImage(account: Account, peerId: PeerId, text: Stri } public func standaloneUploadedFile(account: Account, peerId: PeerId, text: String, source: MultipartUploadSource, thumbnailData: Data? = nil, mimeType: String, attributes: [TelegramMediaFileAttribute], hintFileIsLarge: Bool) -> Signal { - let upload = multipartUpload(network: account.network, postbox: account.postbox, source: source, encrypt: peerId.namespace == Namespaces.Peer.SecretChat, tag: TelegramMediaResourceFetchTag(statsCategory: statsCategoryForFileWithAttributes(attributes)), hintFileSize: nil, hintFileIsLarge: hintFileIsLarge) + let upload = multipartUpload(network: account.network, postbox: account.postbox, source: source, encrypt: peerId.namespace == Namespaces.Peer.SecretChat, tag: TelegramMediaResourceFetchTag(statsCategory: statsCategoryForFileWithAttributes(attributes)), hintFileSize: nil, hintFileIsLarge: hintFileIsLarge, forceNoBigParts: false) |> mapError { _ -> StandaloneUploadMediaError in return .generic } let uploadThumbnail: Signal diff --git a/submodules/TelegramCore/Sources/Themes.swift b/submodules/TelegramCore/Sources/Themes.swift index 5916c8308f..b6af8018a9 100644 --- a/submodules/TelegramCore/Sources/Themes.swift +++ b/submodules/TelegramCore/Sources/Themes.swift @@ -203,7 +203,7 @@ private enum UploadedThemeDataContent { } private func uploadedTheme(postbox: Postbox, network: Network, resource: MediaResource) -> Signal { - return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .file), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .file), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> map { result -> UploadedThemeData in return UploadedThemeData(content: .result(result)) } @@ -213,7 +213,7 @@ private func uploadedTheme(postbox: Postbox, network: Network, resource: MediaRe } private func uploadedThemeThumbnail(postbox: Postbox, network: Network, data: Data) -> Signal { - return multipartUpload(network: network, postbox: postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .data(data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> map { result -> UploadedThemeData in return UploadedThemeData(content: .result(result)) } diff --git a/submodules/TelegramCore/Sources/UploadSecureIdFile.swift b/submodules/TelegramCore/Sources/UploadSecureIdFile.swift index 75a0e27255..de72484f17 100644 --- a/submodules/TelegramCore/Sources/UploadSecureIdFile.swift +++ b/submodules/TelegramCore/Sources/UploadSecureIdFile.swift @@ -105,7 +105,7 @@ public func uploadSecureIdFile(context: SecureIdAccessContext, postbox: Postbox, return .fail(.generic) } - return multipartUpload(network: network, postbox: postbox, source: .data(encryptedData.data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .data(encryptedData.data), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> mapError { _ -> UploadSecureIdFileError in return .generic } diff --git a/submodules/TelegramCore/Sources/Wallpapers.swift b/submodules/TelegramCore/Sources/Wallpapers.swift index 039f88385e..4557a74894 100644 --- a/submodules/TelegramCore/Sources/Wallpapers.swift +++ b/submodules/TelegramCore/Sources/Wallpapers.swift @@ -92,7 +92,7 @@ private enum UploadedWallpaperDataContent { } private func uploadedWallpaper(postbox: Postbox, network: Network, resource: MediaResource) -> Signal { - return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false) + return multipartUpload(network: network, postbox: postbox, source: .resource(.standalone(resource: resource)), encrypt: false, tag: TelegramMediaResourceFetchTag(statsCategory: .image), hintFileSize: nil, hintFileIsLarge: false, forceNoBigParts: false) |> map { result -> UploadedWallpaperData in return UploadedWallpaperData(resource: resource, content: .result(result)) }