From 54b0fbff60a67ef8faa5046c8127edd68b764037 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Fri, 23 Dec 2022 21:51:22 +0400 Subject: [PATCH] Storage improvements --- submodules/Postbox/Sources/MediaBox.swift | 11 +- .../Postbox/Sources/MediaResource.swift | 1 - .../Sources/StorageBox/StorageBox.swift | 298 +++++++++--------- .../Sources/Account/Account.swift | 15 +- .../Resources/CollectCacheUsageStats.swift | 168 +++++++++- .../Resources/TelegramEngineResources.swift | 29 +- 6 files changed, 355 insertions(+), 167 deletions(-) diff --git a/submodules/Postbox/Sources/MediaBox.swift b/submodules/Postbox/Sources/MediaBox.swift index 312e530b44..d93ea02ddc 100644 --- a/submodules/Postbox/Sources/MediaBox.swift +++ b/submodules/Postbox/Sources/MediaBox.swift @@ -1278,7 +1278,7 @@ public final class MediaBox { } } - public func updateResourceIndex(completion: @escaping () -> Void) -> Disposable { + public func updateResourceIndex(lowImpact: Bool, completion: @escaping () -> Void) -> Disposable { let basePath = self.basePath let storageBox = self.storageBox @@ -1318,7 +1318,14 @@ public final class MediaBox { if addedCount != 0 { postboxLog("UpdateResourceIndex: added \(addedCount) unreferenced ids") } - processNext() + + if lowImpact { + processQueue.after(0.4, { + processNext() + }) + } else { + processNext() + } }) } } diff --git a/submodules/Postbox/Sources/MediaResource.swift b/submodules/Postbox/Sources/MediaResource.swift index dcf3bcc7e8..272ef949bd 100644 --- a/submodules/Postbox/Sources/MediaResource.swift +++ b/submodules/Postbox/Sources/MediaResource.swift @@ -55,7 +55,6 @@ public enum MediaResourceUserContentType: UInt8, Equatable { case video = 2 case audio = 3 case file = 4 - case gif = 5 case sticker = 6 case avatar = 7 } diff --git a/submodules/Postbox/Sources/StorageBox/StorageBox.swift b/submodules/Postbox/Sources/StorageBox/StorageBox.swift index 7dbbfbffdf..72ffa8dff6 100644 --- a/submodules/Postbox/Sources/StorageBox/StorageBox.swift +++ b/submodules/Postbox/Sources/StorageBox/StorageBox.swift @@ -154,7 +154,6 @@ public final class StorageBox { let hashIdToInfoTable: ValueBoxTable let idToReferenceTable: ValueBoxTable let peerIdToIdTable: ValueBoxTable - let peerIdTable: ValueBoxTable let peerContentTypeStatsTable: ValueBoxTable let contentTypeStatsTable: ValueBoxTable let metadataTable: ValueBoxTable @@ -179,7 +178,6 @@ public final class StorageBox { self.hashIdToInfoTable = ValueBoxTable(id: 15, keyType: .binary, compactValuesOnCreation: true) self.idToReferenceTable = ValueBoxTable(id: 16, keyType: .binary, compactValuesOnCreation: true) self.peerIdToIdTable = ValueBoxTable(id: 17, keyType: .binary, compactValuesOnCreation: true) - self.peerIdTable = ValueBoxTable(id: 18, keyType: .binary, compactValuesOnCreation: true) self.peerContentTypeStatsTable = ValueBoxTable(id: 19, keyType: .binary, compactValuesOnCreation: true) self.contentTypeStatsTable = ValueBoxTable(id: 20, keyType: .binary, compactValuesOnCreation: true) self.metadataTable = ValueBoxTable(id: 21, keyType: .binary, compactValuesOnCreation: true) @@ -355,37 +353,11 @@ public final class StorageBox { } if !alreadyStored { - var idInPeerIdStored = false - let peerIdIdKey = ValueBoxKey(length: 8 + 16) peerIdIdKey.setInt64(0, value: reference.peerId) peerIdIdKey.setData(8, value: hashId.data) - var peerIdIdCount: Int32 = 0 - if let value = self.valueBox.get(self.peerIdToIdTable, key: peerIdIdKey) { - idInPeerIdStored = true - if value.length == 4 { - memcpy(&peerIdIdCount, value.memory, 4) - } else { - assertionFailure() - } - } - peerIdIdCount += 1 - self.valueBox.set(self.peerIdToIdTable, key: peerIdIdKey, value: MemoryBuffer(memory: &peerIdIdCount, capacity: 4, length: 4, freeWhenDone: false)) - if !idInPeerIdStored { - let peerIdKey = ValueBoxKey(length: 8) - peerIdKey.setInt64(0, value: reference.peerId) - var peerIdCount: Int32 = 0 - if let value = self.valueBox.get(self.peerIdTable, key: peerIdKey) { - if value.length == 4 { - memcpy(&peerIdCount, value.memory, 4) - } else { - assertionFailure() - } - } - peerIdCount += 1 - self.valueBox.set(self.peerIdTable, key: peerIdKey, value: MemoryBuffer(memory: &peerIdCount, capacity: 4, length: 4, freeWhenDone: false)) - } + self.valueBox.setOrIgnore(self.peerIdToIdTable, key: peerIdIdKey, value: MemoryBuffer()) } } @@ -455,73 +427,18 @@ public final class StorageBox { func addEmptyReferencesIfNotReferenced(ids: [(id: Data, size: Int64)], contentType: UInt8) -> Int { self.valueBox.begin() + let mainKey = ValueBoxKey(length: 16) var addedCount = 0 for (id, size) in ids { - let reference = Reference(peerId: 0, messageNamespace: 0, messageId: 0) - let hashId = md5Hash(id) - - let mainKey = ValueBoxKey(length: 16) mainKey.setData(0, value: hashId.data) if self.valueBox.exists(self.hashIdToInfoTable, key: mainKey) { continue } + self.internalAdd(reference: StorageBox.Reference(peerId: 0, messageNamespace: 0, messageId: 0), to: id, contentType: contentType, size: size) addedCount += 1 - - self.valueBox.set(self.hashIdToInfoTable, key: mainKey, value: ItemInfo(id: id, contentType: contentType, size: size).serialize()) - - self.internalAddSize(contentType: contentType, delta: size) - - let idKey = ValueBoxKey(length: hashId.data.count + 8 + 1 + 4) - idKey.setData(0, value: hashId.data) - idKey.setInt64(hashId.data.count, value: reference.peerId) - idKey.setUInt8(hashId.data.count + 8, value: reference.messageNamespace) - idKey.setInt32(hashId.data.count + 8 + 1, value: reference.messageId) - - var alreadyStored = false - if !self.valueBox.exists(self.idToReferenceTable, key: idKey) { - self.valueBox.setOrIgnore(self.idToReferenceTable, key: idKey, value: MemoryBuffer()) - } else { - alreadyStored = true - } - - if !alreadyStored { - var idInPeerIdStored = false - - let peerIdIdKey = ValueBoxKey(length: 8 + 16) - peerIdIdKey.setInt64(0, value: reference.peerId) - peerIdIdKey.setData(8, value: hashId.data) - var peerIdIdCount: Int32 = 0 - if let value = self.valueBox.get(self.peerIdToIdTable, key: peerIdIdKey) { - idInPeerIdStored = true - if value.length == 4 { - memcpy(&peerIdIdCount, value.memory, 4) - } else { - assertionFailure() - } - } - peerIdIdCount += 1 - self.valueBox.set(self.peerIdToIdTable, key: peerIdIdKey, value: MemoryBuffer(memory: &peerIdIdCount, capacity: 4, length: 4, freeWhenDone: false)) - - if !idInPeerIdStored { - let peerIdKey = ValueBoxKey(length: 8) - peerIdKey.setInt64(0, value: reference.peerId) - var peerIdCount: Int32 = 0 - if let value = self.valueBox.get(self.peerIdTable, key: peerIdKey) { - if value.length == 4 { - memcpy(&peerIdCount, value.memory, 4) - } else { - assertionFailure() - } - } - peerIdCount += 1 - self.valueBox.set(self.peerIdTable, key: peerIdKey, value: MemoryBuffer(memory: &peerIdCount, capacity: 4, length: 4, freeWhenDone: false)) - - self.internalAddSize(peerId: 0, contentType: contentType, delta: size) - } - } } self.valueBox.commit() @@ -529,67 +446,52 @@ public final class StorageBox { return addedCount } + private func internalRemove(hashId: Data) { + let mainKey = ValueBoxKey(length: 16) + let peerIdIdKey = ValueBoxKey(length: 8 + 16) + + mainKey.setData(0, value: hashId) + + guard let infoValue = self.valueBox.get(self.hashIdToInfoTable, key: mainKey) else { + return + } + let info = ItemInfo(buffer: infoValue) + self.valueBox.remove(self.hashIdToInfoTable, key: mainKey, secure: false) + + if info.size != 0 { + self.internalAddSize(contentType: info.contentType, delta: -info.size) + } + + var referenceKeys: [ValueBoxKey] = [] + self.valueBox.range(self.idToReferenceTable, start: mainKey, end: mainKey.successor, keys: { key in + referenceKeys.append(key) + return true + }, limit: 0) + var peerIds = Set() + for key in referenceKeys { + peerIds.insert(key.getInt64(16)) + self.valueBox.remove(self.idToReferenceTable, key: key, secure: false) + } + + for peerId in peerIds { + peerIdIdKey.setInt64(0, value: peerId) + peerIdIdKey.setData(8, value: hashId) + + if self.valueBox.exists(self.peerIdToIdTable, key: peerIdIdKey) { + self.valueBox.remove(self.peerIdToIdTable, key: peerIdIdKey, secure: false) + + if info.size != 0 { + self.internalAddSize(peerId: peerId, contentType: info.contentType, delta: -info.size) + } + } + } + } + func remove(ids: [Data]) { self.valueBox.begin() - let mainKey = ValueBoxKey(length: 16) - let peerIdIdKey = ValueBoxKey(length: 8 + 16) - let peerIdKey = ValueBoxKey(length: 8) - for id in ids { - let hashId = md5Hash(id) - mainKey.setData(0, value: hashId.data) - - guard let infoValue = self.valueBox.get(self.hashIdToInfoTable, key: mainKey) else { - continue - } - let info = ItemInfo(buffer: infoValue) - self.valueBox.remove(self.hashIdToInfoTable, key: mainKey, secure: false) - - if info.size != 0 { - self.internalAddSize(contentType: info.contentType, delta: -info.size) - } - - var referenceKeys: [ValueBoxKey] = [] - self.valueBox.range(self.idToReferenceTable, start: mainKey, end: mainKey.successor, keys: { key in - referenceKeys.append(key) - return true - }, limit: 0) - var peerIds = Set() - for key in referenceKeys { - peerIds.insert(key.getInt64(16)) - self.valueBox.remove(self.idToReferenceTable, key: key, secure: false) - } - - for peerId in peerIds { - peerIdIdKey.setInt64(0, value: peerId) - peerIdIdKey.setData(8, value: hashId.data) - - if self.valueBox.exists(self.peerIdToIdTable, key: peerIdIdKey) { - self.valueBox.remove(self.peerIdToIdTable, key: peerIdIdKey, secure: false) - - if info.size != 0 { - self.internalAddSize(peerId: peerId, contentType: info.contentType, delta: -info.size) - } - - peerIdKey.setInt64(0, value: peerId) - if let value = self.valueBox.get(self.peerIdTable, key: peerIdKey) { - var peerIdCount: Int32 = 0 - if value.length == 4 { - memcpy(&peerIdCount, value.memory, 4) - } else { - assertionFailure() - } - - peerIdCount -= 1 - if peerIdCount > 0 { - self.valueBox.set(self.peerIdTable, key: peerIdKey, value: MemoryBuffer(memory: &peerIdCount, capacity: 4, length: 4, freeWhenDone: false)) - } else { - self.valueBox.remove(self.peerIdTable, key: peerIdKey, secure: false) - } - } - } - } + self.internalRemove(hashId: md5Hash(id).data) } self.valueBox.commit() @@ -600,7 +502,7 @@ public final class StorageBox { self.valueBox.begin() - self.valueBox.scan(self.peerIdTable, keys: { key in + self.valueBox.scan(self.peerIdToIdTable, keys: { key in result.append(PeerId(key.getInt64(0))) return true }) @@ -610,9 +512,7 @@ public final class StorageBox { return result } - func all(peerId: PeerId) -> [Data] { - self.valueBox.begin() - + private func allInternal(peerId: PeerId) -> [Data] { var hashIds: [Data] = [] let peerIdIdKey = ValueBoxKey(length: 8) peerIdIdKey.setInt64(0, value: peerId.toInt64()) @@ -631,6 +531,14 @@ public final class StorageBox { } } + return result + } + + func all(peerId: PeerId) -> [Data] { + self.valueBox.begin() + + let result = self.allInternal(peerId: peerId) + self.valueBox.commit() return result @@ -781,6 +689,88 @@ public final class StorageBox { return allStats } + + func remove(peerId: Int64?, contentTypes: [UInt8]) -> [Data] { + var resultIds: [Data] = [] + + self.valueBox.begin() + + var scannedIds: [Data: Data] = [:] + + for contentType in contentTypes { + self.internalAddSize(contentType: contentType, delta: 0) + } + + self.valueBox.scan(self.hashIdToInfoTable, values: { key, value in + let info = ItemInfo(buffer: value) + if !contentTypes.contains(info.contentType) { + return true + } + scannedIds[key.getData(0, length: 16)] = info.id + return true + }) + + if let peerId = peerId { + var filteredHashIds: [Data] = [] + self.valueBox.scan(self.idToReferenceTable, keys: { key in + let id = key.getData(0, length: 16) + if scannedIds[id] == nil { + return true + } + + let itemPeerId = key.getInt64(16) + //let messageNamespace: UInt8 = key.getUInt8(16 + 8) + //let messageId = key.getInt32(16 + 8 + 1) + + if itemPeerId == peerId { + filteredHashIds.append(id) + } + + return true + }) + for hashId in filteredHashIds { + if let id = scannedIds[hashId] { + self.internalRemove(hashId: hashId) + resultIds.append(id) + } + } + } else { + for (hashId, id) in scannedIds { + self.internalRemove(hashId: hashId) + resultIds.append(id) + } + } + + if let peerId = peerId { + let _ = peerId + } else { + + } + + self.valueBox.commit() + + return Array(resultIds) + } + + func remove(peerIds: Set) -> [Data] { + var resultIds: [Data] = [] + + self.valueBox.begin() + + var scannedIds = Set() + for peerId in peerIds { + scannedIds.formUnion(self.allInternal(peerId: peerId)) + } + + for id in scannedIds { + self.internalRemove(hashId: md5Hash(id).data) + resultIds.append(id) + } + + self.valueBox.commit() + + return Array(resultIds) + } } private let queue: Queue @@ -869,4 +859,18 @@ public final class StorageBox { return EmptyDisposable } } + + public func remove(peerId: PeerId?, contentTypes: [UInt8], completion: @escaping ([Data]) -> Void) { + self.impl.with { impl in + let ids = impl.remove(peerId: peerId?.toInt64(), contentTypes: contentTypes) + completion(ids) + } + } + + public func remove(peerIds: Set, completion: @escaping ([Data]) -> Void) { + self.impl.with { impl in + let ids = impl.remove(peerIds: peerIds) + completion(ids) + } + } } diff --git a/submodules/TelegramCore/Sources/Account/Account.swift b/submodules/TelegramCore/Sources/Account/Account.swift index cb7a162ad5..65adff0e40 100644 --- a/submodules/TelegramCore/Sources/Account/Account.swift +++ b/submodules/TelegramCore/Sources/Account/Account.swift @@ -1280,14 +1280,17 @@ public class Account { self.viewTracker.reset() } - public func cleanupTasks() -> Signal { + public func cleanupTasks(lowImpact: Bool) -> Signal { let postbox = self.postbox - return Signal { subscriber in - return postbox.mediaBox.updateResourceIndex(completion: { - subscriber.putCompletion() - }) - } + return _internal_reindexCacheInBackground(account: self, lowImpact: lowImpact) + |> then( + Signal { subscriber in + return postbox.mediaBox.updateResourceIndex(lowImpact: lowImpact, completion: { + subscriber.putCompletion() + }) + } + ) } public func restartContactManagement() { diff --git a/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift b/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift index 789aee73be..fdf198d986 100644 --- a/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift +++ b/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift @@ -250,7 +250,7 @@ func _internal_collectStorageUsageStats(account: Account) -> Signal Signal<[EngineMessage.Id: Message], NoError> { +func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey], existingMessages: [EngineMessage.Id: Message]) -> Signal<[EngineMessage.Id: Message], NoError> { return account.postbox.transaction { transaction -> [EngineMessage.Id: Message] in var result: [EngineMessage.Id: Message] = [:] for (category, value) in stats.categories { @@ -259,7 +259,9 @@ func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageU } for id in value.messages.keys { if result[id] == nil { - if let message = transaction.getMessage(id) { + if let message = existingMessages[id] { + result[id] = message + } else if let message = transaction.getMessage(id) { result[id] = message } } @@ -270,7 +272,155 @@ func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageU } } -func _internal_reindexCacheInBackground(account: Account) -> Signal { +func _internal_clearStorage(account: Account, peerId: EnginePeer.Id?, categories: [StorageUsageStats.CategoryKey]) -> Signal { + let mediaBox = account.postbox.mediaBox + return Signal { subscriber in + mediaBox.storageBox.remove(peerId: peerId, contentTypes: categories.map { item -> UInt8 in + let mappedItem: MediaResourceUserContentType + switch item { + case .photos: + mappedItem = .image + case .videos: + mappedItem = .video + case .files: + mappedItem = .file + case .music: + mappedItem = .audio + case .stickers: + mappedItem = .sticker + case .avatars: + mappedItem = .avatar + case .misc: + mappedItem = .other + } + return mappedItem.rawValue + }, completion: { ids in + var resourceIds: [MediaResourceId] = [] + for id in ids { + if let value = String(data: id, encoding: .utf8) { + resourceIds.append(MediaResourceId(value)) + } + } + let _ = mediaBox.removeCachedResources(resourceIds).start(completed: { + if peerId == nil && categories.contains(.misc) { + let additionalPaths: [String] = [ + "cache", + "animation-cache", + "short-cache", + ] + + for item in additionalPaths { + let fullPath = mediaBox.basePath + "/\(item)" + if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: fullPath), includingPropertiesForKeys: [.isDirectoryKey], options: .skipsSubdirectoryDescendants) { + for url in enumerator { + guard let url = url as? URL else { + continue + } + let _ = try? FileManager.default.removeItem(at: url) + } + } + } + + subscriber.putCompletion() + } else { + subscriber.putCompletion() + } + }) + }) + + return ActionDisposable { + } + } +} + +func _internal_clearStorage(account: Account, peerIds: Set) -> Signal { + let mediaBox = account.postbox.mediaBox + return Signal { subscriber in + mediaBox.storageBox.remove(peerIds: peerIds, completion: { ids in + var resourceIds: [MediaResourceId] = [] + for id in ids { + if let value = String(data: id, encoding: .utf8) { + resourceIds.append(MediaResourceId(value)) + } + } + let _ = mediaBox.removeCachedResources(resourceIds).start(completed: { + subscriber.putCompletion() + }) + }) + + return ActionDisposable { + } + } +} + +func _internal_clearStorage(account: Account, messages: [Message]) -> Signal { + let mediaBox = account.postbox.mediaBox + + return Signal { subscriber in + DispatchQueue.global().async { + var resourceIds = Set() + for message in messages { + for media in message.media { + if let image = media as? TelegramMediaImage { + for representation in image.representations { + resourceIds.insert(representation.resource.id) + } + } else if let file = media as? TelegramMediaFile { + for representation in file.previewRepresentations { + resourceIds.insert(representation.resource.id) + } + resourceIds.insert(file.resource.id) + } else if let webpage = media as? TelegramMediaWebpage { + if case let .Loaded(content) = webpage.content { + if let image = content.image { + for representation in image.representations { + resourceIds.insert(representation.resource.id) + } + } + if let file = content.file { + for representation in file.previewRepresentations { + resourceIds.insert(representation.resource.id) + } + resourceIds.insert(file.resource.id) + } + } + } else if let game = media as? TelegramMediaGame { + if let image = game.image { + for representation in image.representations { + resourceIds.insert(representation.resource.id) + } + } + if let file = game.file { + for representation in file.previewRepresentations { + resourceIds.insert(representation.resource.id) + } + resourceIds.insert(file.resource.id) + } + } + } + } + + var removeIds: [Data] = [] + for resourceId in resourceIds { + if let id = resourceId.stringRepresentation.data(using: .utf8) { + removeIds.append(id) + } + } + + mediaBox.storageBox.remove(ids: removeIds) + let _ = mediaBox.removeCachedResources(Array(resourceIds)).start(completed: { + subscriber.putCompletion() + }) + } + + return ActionDisposable { + } + } +} + +func _internal_reindexCacheInBackground(account: Account, lowImpact: Bool) -> Signal { + let postbox = account.postbox + let queue = Queue(name: "ReindexCacheInBackground") return Signal { subscriber in let isCancelled = Atomic(value: false) @@ -280,7 +430,7 @@ func _internal_reindexCacheInBackground(account: Account) -> Signal (messagesByMediaId: [MediaId: [MessageId]], mediaMap: [MediaId: Media], nextLowerBound: MessageIndex?) in + let _ = (postbox.transaction { transaction -> (messagesByMediaId: [MediaId: [MessageId]], mediaMap: [MediaId: Media], nextLowerBound: MessageIndex?) in return transaction.enumerateMediaMessages(lowerBound: lowerBound, upperBound: nil, limit: 1000) } |> deliverOn(queue)).start(next: { result in @@ -288,7 +438,7 @@ func _internal_reindexCacheInBackground(account: Account) -> Signal Void = { messageIds, resource, contentType in let size = mediaBox.fileSizeForId(resource.id) @@ -352,7 +502,13 @@ func _internal_reindexCacheInBackground(account: Account) -> Signal Signal<[EngineMessage.Id: Message], NoError> { - return _internal_renderStorageUsageStatsMessages(account: self.account, stats: stats, categories: categories) + public func renderStorageUsageStatsMessages(stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey], existingMessages: [EngineMessage.Id: Message]) -> Signal<[EngineMessage.Id: Message], NoError> { + return _internal_renderStorageUsageStatsMessages(account: self.account, stats: stats, categories: categories, existingMessages: existingMessages) + } + + public func clearStorage(peerId: EnginePeer.Id?, categories: [StorageUsageStats.CategoryKey]) -> Signal { + return _internal_clearStorage(account: self.account, peerId: peerId, categories: categories) + } + + public func clearStorage(peerIds: Set) -> Signal { + _internal_clearStorage(account: self.account, peerIds: peerIds) + } + + public func clearStorage(messages: [Message]) -> Signal { + _internal_clearStorage(account: self.account, messages: messages) } public func clearCachedMediaResources(mediaResourceIds: Set) -> Signal { return _internal_clearCachedMediaResources(account: self.account, mediaResourceIds: mediaResourceIds) } - public func reindexCacheInBackground() -> Signal { - return _internal_reindexCacheInBackground(account: self.account) + public func reindexCacheInBackground(lowImpact: Bool) -> Signal { + let mediaBox = self.account.postbox.mediaBox + + return _internal_reindexCacheInBackground(account: self.account, lowImpact: lowImpact) + |> then(Signal { subscriber in + return mediaBox.updateResourceIndex(lowImpact: lowImpact, completion: { + subscriber.putCompletion() + }) + }) } public func data(id: EngineMediaResource.Id, attemptSynchronously: Bool = false) -> Signal {