ThePain/Swiftgram
1
0
Fork 0
mirror of https://github.com/Swiftgram/Telegram-iOS.git synced 2025-10-09 03:20:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Storage improvements

Browse Source
This commit is contained in:
Ali 2022-12-23 21:51:22 +04:00
parent e375192212
commit 54b0fbff60
6 changed files with 355 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
submodules/Postbox/Sources/MediaBox.swift
View File

@ -1278,7 +1278,7 @@ public final class MediaBox {
}
}
public func updateResourceIndex(completion: @escaping () -> Void) -> Disposable {
public func updateResourceIndex(lowImpact: Bool, completion: @escaping () -> Void) -> Disposable {
let basePath = self.basePath
let storageBox = self.storageBox
@ -1318,7 +1318,14 @@ public final class MediaBox {
if addedCount != 0 {
postboxLog("UpdateResourceIndex: added \(addedCount) unreferenced ids")
}
processNext()
if lowImpact {
processQueue.after(0.4, {
processNext()
})
} else {
processNext()
}
})
}
}

1
submodules/Postbox/Sources/MediaResource.swift
View File

@ -55,7 +55,6 @@ public enum MediaResourceUserContentType: UInt8, Equatable {
case video = 2
case audio = 3
case file = 4
case gif = 5
case sticker = 6
case avatar = 7
}

298
submodules/Postbox/Sources/StorageBox/StorageBox.swift
View File

@ -154,7 +154,6 @@ public final class StorageBox {
let hashIdToInfoTable: ValueBoxTable
let idToReferenceTable: ValueBoxTable
let peerIdToIdTable: ValueBoxTable
let peerIdTable: ValueBoxTable
let peerContentTypeStatsTable: ValueBoxTable
let contentTypeStatsTable: ValueBoxTable
let metadataTable: ValueBoxTable
@ -179,7 +178,6 @@ public final class StorageBox {
self.hashIdToInfoTable = ValueBoxTable(id: 15, keyType: .binary, compactValuesOnCreation: true)
self.idToReferenceTable = ValueBoxTable(id: 16, keyType: .binary, compactValuesOnCreation: true)
self.peerIdToIdTable = ValueBoxTable(id: 17, keyType: .binary, compactValuesOnCreation: true)
self.peerIdTable = ValueBoxTable(id: 18, keyType: .binary, compactValuesOnCreation: true)
self.peerContentTypeStatsTable = ValueBoxTable(id: 19, keyType: .binary, compactValuesOnCreation: true)
self.contentTypeStatsTable = ValueBoxTable(id: 20, keyType: .binary, compactValuesOnCreation: true)
self.metadataTable = ValueBoxTable(id: 21, keyType: .binary, compactValuesOnCreation: true)
@ -355,37 +353,11 @@ public final class StorageBox {
}
if !alreadyStored {
var idInPeerIdStored = false
let peerIdIdKey = ValueBoxKey(length: 8 + 16)
peerIdIdKey.setInt64(0, value: reference.peerId)
peerIdIdKey.setData(8, value: hashId.data)
var peerIdIdCount: Int32 = 0
if let value = self.valueBox.get(self.peerIdToIdTable, key: peerIdIdKey) {
idInPeerIdStored = true
if value.length == 4 {
memcpy(&peerIdIdCount, value.memory, 4)
} else {
assertionFailure()
}
}
peerIdIdCount += 1
self.valueBox.set(self.peerIdToIdTable, key: peerIdIdKey, value: MemoryBuffer(memory: &peerIdIdCount, capacity: 4, length: 4, freeWhenDone: false))
if !idInPeerIdStored {
let peerIdKey = ValueBoxKey(length: 8)
peerIdKey.setInt64(0, value: reference.peerId)
var peerIdCount: Int32 = 0
if let value = self.valueBox.get(self.peerIdTable, key: peerIdKey) {
if value.length == 4 {
memcpy(&peerIdCount, value.memory, 4)
} else {
assertionFailure()
}
}
peerIdCount += 1
self.valueBox.set(self.peerIdTable, key: peerIdKey, value: MemoryBuffer(memory: &peerIdCount, capacity: 4, length: 4, freeWhenDone: false))
}
self.valueBox.setOrIgnore(self.peerIdToIdTable, key: peerIdIdKey, value: MemoryBuffer())
}
}
@ -455,73 +427,18 @@ public final class StorageBox {
func addEmptyReferencesIfNotReferenced(ids: [(id: Data, size: Int64)], contentType: UInt8) -> Int {
self.valueBox.begin()
let mainKey = ValueBoxKey(length: 16)
var addedCount = 0
for (id, size) in ids {
let reference = Reference(peerId: 0, messageNamespace: 0, messageId: 0)
let hashId = md5Hash(id)
let mainKey = ValueBoxKey(length: 16)
mainKey.setData(0, value: hashId.data)
if self.valueBox.exists(self.hashIdToInfoTable, key: mainKey) {
continue
}
self.internalAdd(reference: StorageBox.Reference(peerId: 0, messageNamespace: 0, messageId: 0), to: id, contentType: contentType, size: size)
addedCount += 1
self.valueBox.set(self.hashIdToInfoTable, key: mainKey, value: ItemInfo(id: id, contentType: contentType, size: size).serialize())
self.internalAddSize(contentType: contentType, delta: size)
let idKey = ValueBoxKey(length: hashId.data.count + 8 + 1 + 4)
idKey.setData(0, value: hashId.data)
idKey.setInt64(hashId.data.count, value: reference.peerId)
idKey.setUInt8(hashId.data.count + 8, value: reference.messageNamespace)
idKey.setInt32(hashId.data.count + 8 + 1, value: reference.messageId)
var alreadyStored = false
if !self.valueBox.exists(self.idToReferenceTable, key: idKey) {
self.valueBox.setOrIgnore(self.idToReferenceTable, key: idKey, value: MemoryBuffer())
} else {
alreadyStored = true
}
if !alreadyStored {
var idInPeerIdStored = false
let peerIdIdKey = ValueBoxKey(length: 8 + 16)
peerIdIdKey.setInt64(0, value: reference.peerId)
peerIdIdKey.setData(8, value: hashId.data)
var peerIdIdCount: Int32 = 0
if let value = self.valueBox.get(self.peerIdToIdTable, key: peerIdIdKey) {
idInPeerIdStored = true
if value.length == 4 {
memcpy(&peerIdIdCount, value.memory, 4)
} else {
assertionFailure()
}
}
peerIdIdCount += 1
self.valueBox.set(self.peerIdToIdTable, key: peerIdIdKey, value: MemoryBuffer(memory: &peerIdIdCount, capacity: 4, length: 4, freeWhenDone: false))
if !idInPeerIdStored {
let peerIdKey = ValueBoxKey(length: 8)
peerIdKey.setInt64(0, value: reference.peerId)
var peerIdCount: Int32 = 0
if let value = self.valueBox.get(self.peerIdTable, key: peerIdKey) {
if value.length == 4 {
memcpy(&peerIdCount, value.memory, 4)
} else {
assertionFailure()
}
}
peerIdCount += 1
self.valueBox.set(self.peerIdTable, key: peerIdKey, value: MemoryBuffer(memory: &peerIdCount, capacity: 4, length: 4, freeWhenDone: false))
self.internalAddSize(peerId: 0, contentType: contentType, delta: size)
}
}
}
self.valueBox.commit()
@ -529,67 +446,52 @@ public final class StorageBox {
return addedCount
}
private func internalRemove(hashId: Data) {
let mainKey = ValueBoxKey(length: 16)
let peerIdIdKey = ValueBoxKey(length: 8 + 16)
mainKey.setData(0, value: hashId)
guard let infoValue = self.valueBox.get(self.hashIdToInfoTable, key: mainKey) else {
return
}
let info = ItemInfo(buffer: infoValue)
self.valueBox.remove(self.hashIdToInfoTable, key: mainKey, secure: false)
if info.size != 0 {
self.internalAddSize(contentType: info.contentType, delta: -info.size)
}
var referenceKeys: [ValueBoxKey] = []
self.valueBox.range(self.idToReferenceTable, start: mainKey, end: mainKey.successor, keys: { key in
referenceKeys.append(key)
return true
}, limit: 0)
var peerIds = Set<Int64>()
for key in referenceKeys {
peerIds.insert(key.getInt64(16))
self.valueBox.remove(self.idToReferenceTable, key: key, secure: false)
}
for peerId in peerIds {
peerIdIdKey.setInt64(0, value: peerId)
peerIdIdKey.setData(8, value: hashId)
if self.valueBox.exists(self.peerIdToIdTable, key: peerIdIdKey) {
self.valueBox.remove(self.peerIdToIdTable, key: peerIdIdKey, secure: false)
if info.size != 0 {
self.internalAddSize(peerId: peerId, contentType: info.contentType, delta: -info.size)
}
}
}
}
func remove(ids: [Data]) {
self.valueBox.begin()
let mainKey = ValueBoxKey(length: 16)
let peerIdIdKey = ValueBoxKey(length: 8 + 16)
let peerIdKey = ValueBoxKey(length: 8)
for id in ids {
let hashId = md5Hash(id)
mainKey.setData(0, value: hashId.data)
guard let infoValue = self.valueBox.get(self.hashIdToInfoTable, key: mainKey) else {
continue
}
let info = ItemInfo(buffer: infoValue)
self.valueBox.remove(self.hashIdToInfoTable, key: mainKey, secure: false)
if info.size != 0 {
self.internalAddSize(contentType: info.contentType, delta: -info.size)
}
var referenceKeys: [ValueBoxKey] = []
self.valueBox.range(self.idToReferenceTable, start: mainKey, end: mainKey.successor, keys: { key in
referenceKeys.append(key)
return true
}, limit: 0)
var peerIds = Set<Int64>()
for key in referenceKeys {
peerIds.insert(key.getInt64(16))
self.valueBox.remove(self.idToReferenceTable, key: key, secure: false)
}
for peerId in peerIds {
peerIdIdKey.setInt64(0, value: peerId)
peerIdIdKey.setData(8, value: hashId.data)
if self.valueBox.exists(self.peerIdToIdTable, key: peerIdIdKey) {
self.valueBox.remove(self.peerIdToIdTable, key: peerIdIdKey, secure: false)
if info.size != 0 {
self.internalAddSize(peerId: peerId, contentType: info.contentType, delta: -info.size)
}
peerIdKey.setInt64(0, value: peerId)
if let value = self.valueBox.get(self.peerIdTable, key: peerIdKey) {
var peerIdCount: Int32 = 0
if value.length == 4 {
memcpy(&peerIdCount, value.memory, 4)
} else {
assertionFailure()
}
peerIdCount -= 1
if peerIdCount > 0 {
self.valueBox.set(self.peerIdTable, key: peerIdKey, value: MemoryBuffer(memory: &peerIdCount, capacity: 4, length: 4, freeWhenDone: false))
} else {
self.valueBox.remove(self.peerIdTable, key: peerIdKey, secure: false)
}
}
}
}
self.internalRemove(hashId: md5Hash(id).data)
}
self.valueBox.commit()
@ -600,7 +502,7 @@ public final class StorageBox {
self.valueBox.begin()
self.valueBox.scan(self.peerIdTable, keys: { key in
self.valueBox.scan(self.peerIdToIdTable, keys: { key in
result.append(PeerId(key.getInt64(0)))
return true
})
@ -610,9 +512,7 @@ public final class StorageBox {
return result
}
func all(peerId: PeerId) -> [Data] {
self.valueBox.begin()
private func allInternal(peerId: PeerId) -> [Data] {
var hashIds: [Data] = []
let peerIdIdKey = ValueBoxKey(length: 8)
peerIdIdKey.setInt64(0, value: peerId.toInt64())
@ -631,6 +531,14 @@ public final class StorageBox {
}
}
return result
}
func all(peerId: PeerId) -> [Data] {
self.valueBox.begin()
let result = self.allInternal(peerId: peerId)
self.valueBox.commit()
return result
@ -781,6 +689,88 @@ public final class StorageBox {
return allStats
}
func remove(peerId: Int64?, contentTypes: [UInt8]) -> [Data] {
var resultIds: [Data] = []
self.valueBox.begin()
var scannedIds: [Data: Data] = [:]
for contentType in contentTypes {
self.internalAddSize(contentType: contentType, delta: 0)
}
self.valueBox.scan(self.hashIdToInfoTable, values: { key, value in
let info = ItemInfo(buffer: value)
if !contentTypes.contains(info.contentType) {
return true
}
scannedIds[key.getData(0, length: 16)] = info.id
return true
})
if let peerId = peerId {
var filteredHashIds: [Data] = []
self.valueBox.scan(self.idToReferenceTable, keys: { key in
let id = key.getData(0, length: 16)
if scannedIds[id] == nil {
return true
}
let itemPeerId = key.getInt64(16)
//let messageNamespace: UInt8 = key.getUInt8(16 + 8)
//let messageId = key.getInt32(16 + 8 + 1)
if itemPeerId == peerId {
filteredHashIds.append(id)
}
return true
})
for hashId in filteredHashIds {
if let id = scannedIds[hashId] {
self.internalRemove(hashId: hashId)
resultIds.append(id)
}
}
} else {
for (hashId, id) in scannedIds {
self.internalRemove(hashId: hashId)
resultIds.append(id)
}
}
if let peerId = peerId {
let _ = peerId
} else {
}
self.valueBox.commit()
return Array(resultIds)
}
func remove(peerIds: Set<PeerId>) -> [Data] {
var resultIds: [Data] = []
self.valueBox.begin()
var scannedIds = Set<Data>()
for peerId in peerIds {
scannedIds.formUnion(self.allInternal(peerId: peerId))
}
for id in scannedIds {
self.internalRemove(hashId: md5Hash(id).data)
resultIds.append(id)
}
self.valueBox.commit()
return Array(resultIds)
}
}
private let queue: Queue
@ -869,4 +859,18 @@ public final class StorageBox {
return EmptyDisposable
}
}
public func remove(peerId: PeerId?, contentTypes: [UInt8], completion: @escaping ([Data]) -> Void) {
self.impl.with { impl in
let ids = impl.remove(peerId: peerId?.toInt64(), contentTypes: contentTypes)
completion(ids)
}
}
public func remove(peerIds: Set<PeerId>, completion: @escaping ([Data]) -> Void) {
self.impl.with { impl in
let ids = impl.remove(peerIds: peerIds)
completion(ids)
}
}
}

15
submodules/TelegramCore/Sources/Account/Account.swift
View File

@ -1280,14 +1280,17 @@ public class Account {
self.viewTracker.reset()
}
public func cleanupTasks() -> Signal<Never, NoError> {
public func cleanupTasks(lowImpact: Bool) -> Signal<Never, NoError> {
let postbox = self.postbox
return Signal { subscriber in
return postbox.mediaBox.updateResourceIndex(completion: {
subscriber.putCompletion()
})
}
return _internal_reindexCacheInBackground(account: self, lowImpact: lowImpact)
|> then(
Signal { subscriber in
return postbox.mediaBox.updateResourceIndex(lowImpact: lowImpact, completion: {
subscriber.putCompletion()
})
}
)
}
public func restartContactManagement() {

168
submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift
View File

@ -250,7 +250,7 @@ func _internal_collectStorageUsageStats(account: Account) -> Signal<AllStorageUs
}
}
func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey]) -> Signal<[EngineMessage.Id: Message], NoError> {
func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey], existingMessages: [EngineMessage.Id: Message]) -> Signal<[EngineMessage.Id: Message], NoError> {
return account.postbox.transaction { transaction -> [EngineMessage.Id: Message] in
var result: [EngineMessage.Id: Message] = [:]
for (category, value) in stats.categories {
@ -259,7 +259,9 @@ func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageU
}
for id in value.messages.keys {
if result[id] == nil {
if let message = transaction.getMessage(id) {
if let message = existingMessages[id] {
result[id] = message
} else if let message = transaction.getMessage(id) {
result[id] = message
}
}
@ -270,7 +272,155 @@ func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageU
}
}
func _internal_reindexCacheInBackground(account: Account) -> Signal<Never, NoError> {
func _internal_clearStorage(account: Account, peerId: EnginePeer.Id?, categories: [StorageUsageStats.CategoryKey]) -> Signal<Never, NoError> {
let mediaBox = account.postbox.mediaBox
return Signal { subscriber in
mediaBox.storageBox.remove(peerId: peerId, contentTypes: categories.map { item -> UInt8 in
let mappedItem: MediaResourceUserContentType
switch item {
case .photos:
mappedItem = .image
case .videos:
mappedItem = .video
case .files:
mappedItem = .file
case .music:
mappedItem = .audio
case .stickers:
mappedItem = .sticker
case .avatars:
mappedItem = .avatar
case .misc:
mappedItem = .other
}
return mappedItem.rawValue
}, completion: { ids in
var resourceIds: [MediaResourceId] = []
for id in ids {
if let value = String(data: id, encoding: .utf8) {
resourceIds.append(MediaResourceId(value))
}
}
let _ = mediaBox.removeCachedResources(resourceIds).start(completed: {
if peerId == nil && categories.contains(.misc) {
let additionalPaths: [String] = [
"cache",
"animation-cache",
"short-cache",
]
for item in additionalPaths {
let fullPath = mediaBox.basePath + "/\(item)"
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: fullPath), includingPropertiesForKeys: [.isDirectoryKey], options: .skipsSubdirectoryDescendants) {
for url in enumerator {
guard let url = url as? URL else {
continue
}
let _ = try? FileManager.default.removeItem(at: url)
}
}
}
subscriber.putCompletion()
} else {
subscriber.putCompletion()
}
})
})
return ActionDisposable {
}
}
}
func _internal_clearStorage(account: Account, peerIds: Set<EnginePeer.Id>) -> Signal<Never, NoError> {
let mediaBox = account.postbox.mediaBox
return Signal { subscriber in
mediaBox.storageBox.remove(peerIds: peerIds, completion: { ids in
var resourceIds: [MediaResourceId] = []
for id in ids {
if let value = String(data: id, encoding: .utf8) {
resourceIds.append(MediaResourceId(value))
}
}
let _ = mediaBox.removeCachedResources(resourceIds).start(completed: {
subscriber.putCompletion()
})
})
return ActionDisposable {
}
}
}
func _internal_clearStorage(account: Account, messages: [Message]) -> Signal<Never, NoError> {
let mediaBox = account.postbox.mediaBox
return Signal { subscriber in
DispatchQueue.global().async {
var resourceIds = Set<MediaResourceId>()
for message in messages {
for media in message.media {
if let image = media as? TelegramMediaImage {
for representation in image.representations {
resourceIds.insert(representation.resource.id)
}
} else if let file = media as? TelegramMediaFile {
for representation in file.previewRepresentations {
resourceIds.insert(representation.resource.id)
}
resourceIds.insert(file.resource.id)
} else if let webpage = media as? TelegramMediaWebpage {
if case let .Loaded(content) = webpage.content {
if let image = content.image {
for representation in image.representations {
resourceIds.insert(representation.resource.id)
}
}
if let file = content.file {
for representation in file.previewRepresentations {
resourceIds.insert(representation.resource.id)
}
resourceIds.insert(file.resource.id)
}
}
} else if let game = media as? TelegramMediaGame {
if let image = game.image {
for representation in image.representations {
resourceIds.insert(representation.resource.id)
}
}
if let file = game.file {
for representation in file.previewRepresentations {
resourceIds.insert(representation.resource.id)
}
resourceIds.insert(file.resource.id)
}
}
}
}
var removeIds: [Data] = []
for resourceId in resourceIds {
if let id = resourceId.stringRepresentation.data(using: .utf8) {
removeIds.append(id)
}
}
mediaBox.storageBox.remove(ids: removeIds)
let _ = mediaBox.removeCachedResources(Array(resourceIds)).start(completed: {
subscriber.putCompletion()
})
}
return ActionDisposable {
}
}
}
func _internal_reindexCacheInBackground(account: Account, lowImpact: Bool) -> Signal<Never, NoError> {
let postbox = account.postbox
let queue = Queue(name: "ReindexCacheInBackground")
return Signal { subscriber in
let isCancelled = Atomic<Bool>(value: false)
@ -280,7 +430,7 @@ func _internal_reindexCacheInBackground(account: Account) -> Signal<Never, NoErr
return
}
let _ = (account.postbox.transaction { transaction -> (messagesByMediaId: [MediaId: [MessageId]], mediaMap: [MediaId: Media], nextLowerBound: MessageIndex?) in
let _ = (postbox.transaction { transaction -> (messagesByMediaId: [MediaId: [MessageId]], mediaMap: [MediaId: Media], nextLowerBound: MessageIndex?) in
return transaction.enumerateMediaMessages(lowerBound: lowerBound, upperBound: nil, limit: 1000)
}
|> deliverOn(queue)).start(next: { result in
@ -288,7 +438,7 @@ func _internal_reindexCacheInBackground(account: Account) -> Signal<Never, NoErr
var storageItems: [(reference: StorageBox.Reference, id: Data, contentType: UInt8, size: Int64)] = []
let mediaBox = account.postbox.mediaBox
let mediaBox = postbox.mediaBox
let processResource: ([MessageId], MediaResource, MediaResourceUserContentType) -> Void = { messageIds, resource, contentType in
let size = mediaBox.fileSizeForId(resource.id)
@ -352,7 +502,13 @@ func _internal_reindexCacheInBackground(account: Account) -> Signal<Never, NoErr
}
if let nextLowerBound = result.nextLowerBound {
process(lowerBound: nextLowerBound)
if lowImpact {
queue.after(0.4, {
process(lowerBound: nextLowerBound)
})
} else {
process(lowerBound: nextLowerBound)
}
} else {
subscriber.putCompletion()
}

29
submodules/TelegramCore/Sources/TelegramEngine/Resources/TelegramEngineResources.swift
View File

@ -16,7 +16,7 @@ public extension MediaResourceUserContentType {
self = .sticker
} else if file.isVideo {
if file.isAnimated {
self = .gif
self = .other
} else {
self = .video
}
@ -227,16 +227,35 @@ public extension TelegramEngine {
return _internal_collectStorageUsageStats(account: self.account)
}
public func renderStorageUsageStatsMessages(stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey]) -> Signal<[EngineMessage.Id: Message], NoError> {
return _internal_renderStorageUsageStatsMessages(account: self.account, stats: stats, categories: categories)
public func renderStorageUsageStatsMessages(stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey], existingMessages: [EngineMessage.Id: Message]) -> Signal<[EngineMessage.Id: Message], NoError> {
return _internal_renderStorageUsageStatsMessages(account: self.account, stats: stats, categories: categories, existingMessages: existingMessages)
}
public func clearStorage(peerId: EnginePeer.Id?, categories: [StorageUsageStats.CategoryKey]) -> Signal<Never, NoError> {
return _internal_clearStorage(account: self.account, peerId: peerId, categories: categories)
}
public func clearStorage(peerIds: Set<EnginePeer.Id>) -> Signal<Never, NoError> {
_internal_clearStorage(account: self.account, peerIds: peerIds)
}
public func clearStorage(messages: [Message]) -> Signal<Never, NoError> {
_internal_clearStorage(account: self.account, messages: messages)
}
public func clearCachedMediaResources(mediaResourceIds: Set<MediaResourceId>) -> Signal<Float, NoError> {
return _internal_clearCachedMediaResources(account: self.account, mediaResourceIds: mediaResourceIds)
}
public func reindexCacheInBackground() -> Signal<Never, NoError> {
return _internal_reindexCacheInBackground(account: self.account)
public func reindexCacheInBackground(lowImpact: Bool) -> Signal<Never, NoError> {
let mediaBox = self.account.postbox.mediaBox
return _internal_reindexCacheInBackground(account: self.account, lowImpact: lowImpact)
|> then(Signal { subscriber in
return mediaBox.updateResourceIndex(lowImpact: lowImpact, completion: {
subscriber.putCompletion()
})
})
}
public func data(id: EngineMediaResource.Id, attemptSynchronously: Bool = false) -> Signal<EngineMediaResource.ResourceData, NoError> {