Improve media index

Telegram/BUILD

@@ -1793,6 +1793,7 @@ plist_fragment(
     <key>BGTaskSchedulerPermittedIdentifiers</key>
     <array>
         <string>{telegram_bundle_id}.refresh</string>
+        <string>{telegram_bundle_id}.cleanup</string>
     </array>
     <key>CFBundleAllowMixedLocalizations</key>
     <true/>
@@ -1893,6 +1894,7 @@ plist_fragment(
         <string>location</string>
         <string>remote-notification</string>
         <string>voip</string>
+        <string>processing</string>
     </array>
     <key>UIDeviceFamily</key>
     <array>

Telegram/NotificationService/Sources/NotificationService.swift

@@ -1169,6 +1169,12 @@ private final class NotificationServiceHandler {
                                                             parameters: MediaResourceFetchParameters(
                                                                 tag: nil,
                                                                 info: resourceFetchInfo(resource: resource),
+                                                                location: messageId.flatMap { messageId in
+                                                                    return MediaResourceStorageLocation(
+                                                                        peerId: peerId,
+                                                                        messageId: messageId
+                                                                    )
+                                                                },
                                                                 isRandomAccessAllowed: true
                                                             ),
                                                             encryptionKey: nil,
@@ -1220,6 +1226,7 @@ private final class NotificationServiceHandler {
                                                             parameters: MediaResourceFetchParameters(
                                                                 tag: nil,
                                                                 info: resourceFetchInfo(resource: resource),
+                                                                location: nil,
                                                                 isRandomAccessAllowed: true
                                                             ),
                                                             encryptionKey: nil,

submodules/ContextUI/Sources/ContextActionsContainerNode.swift

@@ -586,7 +586,7 @@ final class InnerTextSelectionTipContainerNode: ASDisplayNode {
         
         let textRightInset: CGFloat
         if let _ = self.iconNode.image {
-            textRightInset = iconSize.width - 8.0
+            textRightInset = iconSize.width - 2.0
         } else {
             textRightInset = 0.0
         }

submodules/Postbox/BUILD

@@ -17,6 +17,7 @@ swift_library(
         "//submodules/StringTransliteration:StringTransliteration",
         "//submodules/ManagedFile:ManagedFile",
         "//submodules/Utils/RangeSet:RangeSet",
+        "//submodules/CryptoUtils",
     ],
     visibility = [
         "//visibility:public",

submodules/Postbox/Sources/MediaBox.swift

@@ -144,6 +144,8 @@ public final class MediaBox {
     private let cacheQueue = Queue()
     private let timeBasedCleanup: TimeBasedCleanup
     
+    public let storageBox: StorageBox
+    
     private let didRemoveResourcesPipe = ValuePipe<Void>()
     public var didRemoveResources: Signal<Void, NoError> {
         return .single(Void()) |> then(self.didRemoveResourcesPipe.signal())
@@ -187,6 +189,10 @@ public final class MediaBox {
     public init(basePath: String) {
         self.basePath = basePath
         
+        self.storageBox = StorageBox(logger: StorageBox.Logger(impl: { string in
+            postboxLog(string)
+        }), basePath: basePath + "/storage")
+        
         self.timeBasedCleanup = TimeBasedCleanup(generalPaths: [
             self.basePath,
             self.basePath + "/cache",
@@ -204,6 +210,14 @@ public final class MediaBox {
         self.timeBasedCleanup.setMaxStoreTimes(general: general, shortLived: shortLived, gigabytesLimit: gigabytesLimit)
     }
     
+    private func idForFileName(name: String) -> String {
+        if name.hasSuffix("_partial") {
+            return String(name[name.startIndex ..< name.index(name.endIndex, offsetBy: -8)])
+        } else {
+            return name
+        }
+    }
+    
     private func fileNameForId(_ id: MediaResourceId) -> String {
         return "\(id.stringRepresentation)"
     }
@@ -580,6 +594,10 @@ public final class MediaBox {
                     return
                 }
                 
+                if let location = parameters?.location {
+                    self.storageBox.add(reference: StorageBox.Reference(peerId: location.peerId, messageNamespace: UInt8(clamping: location.messageId.namespace), messageId: location.messageId.id), to: resource.id.stringRepresentation.data(using: .utf8)!)
+                }
+                
                 guard let (fileContext, releaseContext) = self.fileContext(for: resource.id) else {
                     subscriber.putCompletion()
                     return
@@ -742,6 +760,10 @@ public final class MediaBox {
             self.dataQueue.async {
                 let paths = self.storePathsForId(resource.id)
                 
+                if let location = parameters?.location {
+                    self.storageBox.add(reference: StorageBox.Reference(peerId: location.peerId, messageNamespace: UInt8(clamping: location.messageId.namespace), messageId: location.messageId.id), to: resource.id.stringRepresentation.data(using: .utf8)!)
+                }
+                
                 if let _ = fileSize(paths.complete) {
                     if implNext {
                         subscriber.putNext(.local)
@@ -1174,6 +1196,17 @@ public final class MediaBox {
         }
     }
     
+    public func resourceUsage(id: MediaResourceId) -> Int64 {
+        let paths = self.storePathsForId(id)
+        if let size = fileSize(paths.complete) {
+            return Int64(size)
+        } else if let size = fileSize(paths.partial, useTotalFileAllocatedSize: true) {
+            return Int64(size)
+        } else {
+            return 0
+        }
+    }
+    
     public func collectResourceCacheUsage(_ ids: [MediaResourceId]) -> Signal<[MediaResourceId: Int64], NoError> {
         return Signal { subscriber in
             self.dataQueue.async {
@@ -1194,6 +1227,99 @@ public final class MediaBox {
         }
     }
     
+    public func collectAllResourceUsage() -> Signal<[(id: String?, path: String, size: Int64)], NoError> {
+        return Signal { subscriber in
+            self.dataQueue.async {
+                var result: [(id: String?, path: String, size: Int64)] = []
+                
+                var fileIds = Set<Data>()
+                
+                if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: self.basePath), includingPropertiesForKeys: [.fileSizeKey, .fileResourceIdentifierKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
+                    loop: for url in enumerator {
+                        if let url = url as? URL {
+                            if let fileId = (try? url.resourceValues(forKeys: Set([.fileResourceIdentifierKey])))?.fileResourceIdentifier as? Data {
+                                if fileIds.contains(fileId) {
+                                    //paths.append(url.lastPathComponent)
+                                    continue loop
+                                }
+                            
+                                if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
+                                    fileIds.insert(fileId)
+                                    result.append((id: self.idForFileName(name: url.lastPathComponent), path: url.lastPathComponent, size: Int64(value)))
+                                    //paths.append(url.lastPathComponent)
+                                }
+                            }
+                        }
+                    }
+                }
+                
+                /*var cacheResult: Int64 = 0
+                
+                var excludePrefixes = Set<String>()
+                for id in excludeIds {
+                    let cachedRepresentationPrefix = self.fileNameForId(id)
+                    
+                    excludePrefixes.insert(cachedRepresentationPrefix)
+                }
+                
+                if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: self.basePath + "/cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
+                    loop: for url in enumerator {
+                        if let url = url as? URL {
+                            if let prefix = url.lastPathComponent.components(separatedBy: ":").first, excludePrefixes.contains(prefix) {
+                                continue loop
+                            }
+                            
+                            if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
+                                paths.append("cache/" + url.lastPathComponent)
+                                cacheResult += Int64(value)
+                            }
+                        }
+                    }
+                }
+                
+                func processRecursive(directoryPath: String, subdirectoryPath: String) {
+                    if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: directoryPath), includingPropertiesForKeys: [.fileSizeKey, .isDirectoryKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
+                        loop: for url in enumerator {
+                            if let url = url as? URL {
+                                if let prefix = url.lastPathComponent.components(separatedBy: ":").first, excludePrefixes.contains(prefix) {
+                                    continue loop
+                                }
+                                
+                                if let isDirectory = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectory {
+                                    processRecursive(directoryPath: url.path, subdirectoryPath: subdirectoryPath + "/\(url.lastPathComponent)")
+                                } else if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
+                                    paths.append("\(subdirectoryPath)/" + url.lastPathComponent)
+                                    cacheResult += Int64(value)
+                                }
+                            }
+                        }
+                    }
+                }
+                
+                processRecursive(directoryPath: self.basePath + "/animation-cache", subdirectoryPath: "animation-cache")
+                
+                if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: self.basePath + "/short-cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
+                    loop: for url in enumerator {
+                        if let url = url as? URL {
+                            if let prefix = url.lastPathComponent.components(separatedBy: ":").first, excludePrefixes.contains(prefix) {
+                                continue loop
+                            }
+                            
+                            if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
+                                paths.append("short-cache/" + url.lastPathComponent)
+                                cacheResult += Int64(value)
+                            }
+                        }
+                    }
+                }*/
+                
+                subscriber.putNext(result)
+                subscriber.putCompletion()
+            }
+            return EmptyDisposable
+        }
+    }
+    
     public func collectOtherResourceUsage(excludeIds: Set<MediaResourceId>, combinedExcludeIds: Set<MediaResourceId>) -> Signal<(Int64, [String], Int64), NoError> {
         return Signal { subscriber in
             self.dataQueue.async {

submodules/Postbox/Sources/MediaResource.swift

@@ -39,14 +39,26 @@ public protocol MediaResourceFetchTag {
 public protocol MediaResourceFetchInfo {
 }
 
+public final class MediaResourceStorageLocation {
+    public let peerId: PeerId
+    public let messageId: MessageId
+    
+    public init(peerId: PeerId, messageId: MessageId) {
+        self.peerId = peerId
+        self.messageId = messageId
+    }
+}
+
 public struct MediaResourceFetchParameters {
     public let tag: MediaResourceFetchTag?
     public let info: MediaResourceFetchInfo?
+    public let location: MediaResourceStorageLocation?
     public let isRandomAccessAllowed: Bool
     
-    public init(tag: MediaResourceFetchTag?, info: MediaResourceFetchInfo?, isRandomAccessAllowed: Bool) {
+    public init(tag: MediaResourceFetchTag?, info: MediaResourceFetchInfo?, location: MediaResourceStorageLocation?, isRandomAccessAllowed: Bool) {
         self.tag = tag
         self.info = info
+        self.location = location
         self.isRandomAccessAllowed = isRandomAccessAllowed
     }
 }

submodules/Postbox/Sources/SqliteValueBox.swift

@@ -188,6 +188,8 @@ public final class SqliteValueBox: ValueBox {
     private var updateStatements: [Int32 : SqlitePreparedStatement] = [:]
     private var insertOrReplacePrimaryKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
     private var insertOrReplaceIndexKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
+    private var insertOrIgnorePrimaryKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
+    private var insertOrIgnoreIndexKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
     private var deleteStatements: [Int32 : SqlitePreparedStatement] = [:]
     private var moveStatements: [Int32 : SqlitePreparedStatement] = [:]
     private var copyStatements: [TablePairKey : SqlitePreparedStatement] = [:]
@@ -1220,6 +1222,59 @@ public final class SqliteValueBox: ValueBox {
         return resultStatement
     }
     
+    private func insertOrIgnoreStatement(_ table: SqliteValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement {
+        precondition(self.queue.isCurrent())
+        checkTableKey(table.table, key)
+        
+        let resultStatement: SqlitePreparedStatement
+        
+        if table.table.keyType == .int64 || table.hasPrimaryKey {
+            if let statement = self.insertOrIgnorePrimaryKeyStatements[table.table.id] {
+                resultStatement = statement
+            } else {
+                var statement: OpaquePointer? = nil
+                let status = sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table.table.id) (key, value) VALUES(?, ?) ON CONFLICT(key) DO NOTHING", -1, &statement, nil)
+                if status != SQLITE_OK {
+                    let errorText = self.database.currentError() ?? "Unknown error"
+                    preconditionFailure(errorText)
+                }
+                let preparedStatement = SqlitePreparedStatement(statement: statement)
+                self.insertOrIgnorePrimaryKeyStatements[table.table.id] = preparedStatement
+                resultStatement = preparedStatement
+            }
+        } else {
+            if let statement = self.insertOrIgnoreIndexKeyStatements[table.table.id] {
+                resultStatement = statement
+            } else {
+                var statement: OpaquePointer? = nil
+                let status = sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table.table.id) (key, value) VALUES(?, ?)", -1, &statement, nil)
+                if status != SQLITE_OK {
+                    let errorText = self.database.currentError() ?? "Unknown error"
+                    preconditionFailure(errorText)
+                }
+                let preparedStatement = SqlitePreparedStatement(statement: statement)
+                self.insertOrIgnorePrimaryKeyStatements[table.table.id] = preparedStatement
+                resultStatement = preparedStatement
+            }
+        }
+        
+        resultStatement.reset()
+        
+        switch table.table.keyType {
+            case .binary:
+                resultStatement.bind(1, data: key.memory, length: key.length)
+            case .int64:
+                resultStatement.bind(1, number: key.getInt64(0))
+        }
+        if value.length == 0 {
+            resultStatement.bindNull(2)
+        } else {
+            resultStatement.bind(2, data: value.memory, length: value.length)
+        }
+        
+        return resultStatement
+    }
+    
     private func deleteStatement(_ table: ValueBoxTable, key: ValueBoxKey) -> SqlitePreparedStatement {
         precondition(self.queue.isCurrent())
         checkTableKey(table, key)
@@ -1847,6 +1902,30 @@ public final class SqliteValueBox: ValueBox {
         }
     }
     
+    public func setOrIgnore(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) {
+        precondition(self.queue.isCurrent())
+        let sqliteTable = self.checkTable(table)
+        
+        if sqliteTable.hasPrimaryKey {
+            let statement = self.insertOrIgnoreStatement(sqliteTable, key: key, value: value)
+            while statement.step(handle: self.database.handle, pathToRemoveOnError: self.removeDatabaseOnError ? self.databasePath : nil) {
+            }
+            statement.reset()
+        } else {
+            if self.exists(table, key: key) {
+                let statement = self.updateStatement(table, key: key, value: value)
+                while statement.step(handle: self.database.handle, pathToRemoveOnError: self.removeDatabaseOnError ? self.databasePath : nil) {
+                }
+                statement.reset()
+            } else {
+                let statement = self.insertOrReplaceStatement(sqliteTable, key: key, value: value)
+                while statement.step(handle: self.database.handle, pathToRemoveOnError: self.removeDatabaseOnError ? self.databasePath : nil) {
+                }
+                statement.reset()
+            }
+        }
+    }
+    
     public func remove(_ table: ValueBoxTable, key: ValueBoxKey, secure: Bool) {
         precondition(self.queue.isCurrent())
         if let _ = self.tables[table.id] {
@@ -2108,6 +2187,16 @@ public final class SqliteValueBox: ValueBox {
         }
         self.insertOrReplacePrimaryKeyStatements.removeAll()
         
+        for (_, statement) in self.insertOrIgnoreIndexKeyStatements {
+            statement.destroy()
+        }
+        self.insertOrIgnoreIndexKeyStatements.removeAll()
+        
+        for (_, statement) in self.insertOrIgnorePrimaryKeyStatements {
+            statement.destroy()
+        }
+        self.insertOrIgnorePrimaryKeyStatements.removeAll()
+        
         for (_, statement) in self.deleteStatements {
             statement.destroy()
         }

submodules/Postbox/Sources/StorageBox/StorageBox.swift

@@ -0,0 +1,205 @@
+import Foundation
+import SwiftSignalKit
+import CryptoUtils
+
+public struct HashId: Hashable {
+    public let data: Data
+    
+    public init(data: Data) {
+        precondition(data.count == 16)
+        self.data = data
+    }
+}
+
+private func md5Hash(_ data: Data) -> HashId {
+    let hashData = data.withUnsafeBytes { bytes -> Data in
+        return CryptoMD5(bytes.baseAddress!, Int32(bytes.count))
+    }
+    return HashId(data: hashData)
+}
+
+public final class StorageBox {
+    public struct Reference {
+        public var peerId: PeerId
+        public var messageNamespace: UInt8
+        public var messageId: Int32
+        
+        public init(peerId: PeerId, messageNamespace: UInt8, messageId: Int32) {
+            self.peerId = peerId
+            self.messageNamespace = messageNamespace
+            self.messageId = messageId
+        }
+    }
+    
+    public final class Entry {
+        public let id: Data
+        public let references: [Reference]
+        
+        init(id: Data, references: [Reference]) {
+            self.id = id
+            self.references = references
+        }
+    }
+    
+    public final class Logger {
+        private let impl: (String) -> Void
+        
+        public init(impl: @escaping (String) -> Void) {
+            self.impl = impl
+        }
+        
+        func log(_ string: @autoclosure () -> String) {
+            self.impl(string())
+        }
+    }
+    
+    private final class Impl {
+        let queue: Queue
+        let logger: StorageBox.Logger
+        let basePath: String
+        let valueBox: SqliteValueBox
+        let hashIdToIdTable: ValueBoxTable
+        let idToReferenceTable: ValueBoxTable
+        
+        init(queue: Queue, logger: StorageBox.Logger, basePath: String) {
+            self.queue = queue
+            self.logger = logger
+            self.basePath = basePath
+            
+            let databasePath = self.basePath + "/db"
+            let _ = try? FileManager.default.createDirectory(atPath: databasePath, withIntermediateDirectories: true)
+            var valueBox = SqliteValueBox(basePath: databasePath, queue: queue, isTemporary: false, isReadOnly: false, useCaches: true, removeDatabaseOnError: true, encryptionParameters: nil, upgradeProgress: { _ in })
+            if valueBox == nil {
+                let _ = try? FileManager.default.removeItem(atPath: databasePath)
+                valueBox = SqliteValueBox(basePath: databasePath, queue: queue, isTemporary: false, isReadOnly: false, useCaches: true, removeDatabaseOnError: true, encryptionParameters: nil, upgradeProgress: { _ in })
+            }
+            guard let valueBox else {
+                preconditionFailure("Could not open database")
+            }
+            self.valueBox = valueBox
+            
+            self.hashIdToIdTable = ValueBoxTable(id: 5, keyType: .binary, compactValuesOnCreation: true)
+            self.idToReferenceTable = ValueBoxTable(id: 6, keyType: .binary, compactValuesOnCreation: true)
+        }
+        
+        func add(reference: Reference, to id: Data) {
+            self.valueBox.begin()
+            
+            let hashId = md5Hash(id)
+            
+            let mainKey = ValueBoxKey(length: hashId.data.count)
+            self.valueBox.setOrIgnore(self.hashIdToIdTable, key: mainKey, value: MemoryBuffer(data: id))
+            
+            let idKey = ValueBoxKey(length: hashId.data.count + 8 + 1 + 4)
+            idKey.setData(0, value: hashId.data)
+            idKey.setInt64(hashId.data.count, value: reference.peerId.toInt64())
+            idKey.setUInt8(hashId.data.count + 8, value: reference.messageNamespace)
+            idKey.setInt32(hashId.data.count + 8 + 1, value: reference.messageId)
+            self.valueBox.setOrIgnore(self.idToReferenceTable, key: idKey, value: MemoryBuffer())
+            
+            self.valueBox.commit()
+        }
+        
+        func all() -> [Entry] {
+            var result: [Entry] = []
+            
+            self.valueBox.begin()
+            
+            var currentId: Data?
+            var currentReferences: [Reference] = []
+            
+            self.valueBox.scan(self.idToReferenceTable, keys: { key in
+                let id = key.getData(0, length: 16)
+                
+                let peerId = PeerId(key.getInt64(16))
+                let messageNamespace: UInt8 = key.getUInt8(16 + 8)
+                let messageId = key.getInt32(16 + 8 + 1)
+                
+                let reference = Reference(peerId: peerId, messageNamespace: messageNamespace, messageId: messageId)
+                
+                if currentId == id {
+                    currentReferences.append(reference)
+                } else {
+                    if let currentId = currentId, !currentReferences.isEmpty {
+                        result.append(StorageBox.Entry(id: currentId, references: currentReferences))
+                        currentReferences.removeAll(keepingCapacity: true)
+                    }
+                    currentId = id
+                    currentReferences.append(reference)
+                }
+                
+                return true
+            })
+            
+            self.valueBox.commit()
+            
+            return result
+        }
+        
+        func get(ids: [Data]) -> [Entry] {
+            var result: [Entry] = []
+            
+            self.valueBox.begin()
+            
+            let idKey = ValueBoxKey(length: 16)
+            
+            for id in ids {
+                let hashId = md5Hash(id)
+                idKey.setData(0, value: hashId.data)
+                var currentReferences: [Reference] = []
+                self.valueBox.range(self.idToReferenceTable, start: idKey, end: idKey.successor, keys: { key in
+                    let peerId = PeerId(key.getInt64(16))
+                    let messageNamespace: UInt8 = key.getUInt8(16 + 8)
+                    let messageId = key.getInt32(16 + 8 + 1)
+                    
+                    let reference = Reference(peerId: peerId, messageNamespace: messageNamespace, messageId: messageId)
+                    
+                    currentReferences.append(reference)
+                    return true
+                }, limit: 0)
+                
+                if !currentReferences.isEmpty {
+                    result.append(StorageBox.Entry(id: id, references: currentReferences))
+                }
+            }
+            
+            self.valueBox.commit()
+            
+            return result
+        }
+    }
+    
+    private let queue: Queue
+    private let impl: QueueLocalObject<Impl>
+    
+    public init(queue: Queue = Queue(name: "StorageBox"), logger: StorageBox.Logger, basePath: String) {
+        self.queue = queue
+        self.impl = QueueLocalObject(queue: queue, generate: {
+            return Impl(queue: queue, logger: logger, basePath: basePath)
+        })
+    }
+    
+    public func add(reference: Reference, to id: Data) {
+        self.impl.with { impl in
+            impl.add(reference: reference, to: id)
+        }
+    }
+    
+    public func all() -> Signal<[Entry], NoError> {
+        return self.impl.signalWith { impl, subscriber in
+            subscriber.putNext(impl.all())
+            subscriber.putCompletion()
+            
+            return EmptyDisposable
+        }
+    }
+    
+    public func get(ids: [Data]) -> Signal<[Entry], NoError> {
+        return self.impl.signalWith { impl, subscriber in
+            subscriber.putNext(impl.get(ids: ids))
+            subscriber.putCompletion()
+            
+            return EmptyDisposable
+        }
+    }
+}

submodules/Postbox/Sources/ValueBoxKey.swift

@@ -83,6 +83,15 @@ public struct ValueBoxKey: Equatable, Hashable, CustomStringConvertible, Compara
         memcpy(self.memory + offset, &varValue, 2)
     }
     
+    public func getData(_ offset: Int, length: Int) -> Data {
+        assert(offset >= 0 && offset + length <= self.length)
+        var value = Data(count: length)
+        let _ = value.withUnsafeMutableBytes { bytes in
+            memcpy(bytes.baseAddress!.assumingMemoryBound(to: UInt8.self), self.memory + offset, length)
+        }
+        return value
+    }
+    
     public func getInt32(_ offset: Int) -> Int32 {
         assert(offset >= 0 && offset + 4 <= self.length)
         var value: Int32 = 0

submodules/TelegramCore/Sources/Network/FetchedMediaResource.swift

@@ -29,6 +29,26 @@ public func fetchedMediaResource(mediaBox: MediaBox, reference: MediaResourceRef
     return fetchedMediaResource(mediaBox: mediaBox, reference: reference, ranges: range.flatMap({ [$0] }), statsCategory: statsCategory, reportResultStatus: reportResultStatus, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground)
 }
 
+public extension MediaResourceStorageLocation {
+    convenience init?(reference: MediaResourceReference) {
+        switch reference {
+        case let .media(media, _):
+            switch media {
+            case let .message(message, _):
+                if let id = message.id {
+                    self.init(peerId: id.peerId, messageId: id)
+                } else {
+                    return nil
+                }
+            default:
+                return nil
+            }
+        default:
+            return nil
+        }
+    }
+}
+
 public func fetchedMediaResource(mediaBox: MediaBox, reference: MediaResourceReference, ranges: [(Range<Int64>, MediaBoxFetchPriority)]?, statsCategory: MediaResourceStatsCategory = .generic, reportResultStatus: Bool = false, preferBackgroundReferenceRevalidation: Bool = false, continueInBackground: Bool = false) -> Signal<FetchResourceSourceType, FetchResourceError> {
     var isRandomAccessAllowed = true
     switch reference {
@@ -44,14 +64,24 @@ public func fetchedMediaResource(mediaBox: MediaBox, reference: MediaResourceRef
     
     if let ranges = ranges {
         let signals = ranges.map { (range, priority) -> Signal<Void, FetchResourceError> in
-            return mediaBox.fetchedResourceData(reference.resource, in: range, priority: priority, parameters: MediaResourceFetchParameters(tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory), info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground), isRandomAccessAllowed: isRandomAccessAllowed))
+            return mediaBox.fetchedResourceData(reference.resource, in: range, priority: priority, parameters: MediaResourceFetchParameters(
+                tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory),
+                info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground),
+                location: MediaResourceStorageLocation(reference: reference),
+                isRandomAccessAllowed: isRandomAccessAllowed
+            ))
         }
         return combineLatest(signals)
         |> ignoreValues
         |> map { _ -> FetchResourceSourceType in }
         |> then(.single(.local))
     } else {
-        return mediaBox.fetchedResource(reference.resource, parameters: MediaResourceFetchParameters(tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory), info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground), isRandomAccessAllowed: isRandomAccessAllowed), implNext: reportResultStatus)
+        return mediaBox.fetchedResource(reference.resource, parameters: MediaResourceFetchParameters(
+            tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory),
+            info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground),
+            location: MediaResourceStorageLocation(reference: reference),
+            isRandomAccessAllowed: isRandomAccessAllowed
+        ), implNext: reportResultStatus)
     }
 }
 

submodules/TelegramCore/Sources/State/Fetch.swift

@@ -75,7 +75,12 @@ func fetchResource(account: Account, resource: MediaResource, intervals: Signal<
                 return .fail(.generic)
             }
             return .single(.dataPart(resourceOffset: 0, data: Data(), range: 0 ..< 0, complete: false))
-            |> then(fetchCloudMediaLocation(account: account, resource: cloudResource, datacenterId: cloudResource.datacenterId, size: resource.size == 0 ? nil : resource.size, intervals: intervals, parameters: MediaResourceFetchParameters(tag: nil, info: TelegramCloudMediaResourceFetchInfo(reference: .standalone(resource: file.file.resource), preferBackgroundReferenceRevalidation: false, continueInBackground: false), isRandomAccessAllowed: true)))
+            |> then(fetchCloudMediaLocation(account: account, resource: cloudResource, datacenterId: cloudResource.datacenterId, size: resource.size == 0 ? nil : resource.size, intervals: intervals, parameters: MediaResourceFetchParameters(
+                tag: nil,
+                info: TelegramCloudMediaResourceFetchInfo(reference: .standalone(resource: file.file.resource), preferBackgroundReferenceRevalidation: false, continueInBackground: false),
+                location: nil,
+                isRandomAccessAllowed: true
+            )))
         }
     }
     return nil

submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift

@@ -53,6 +53,203 @@ private final class CacheUsageStatsState {
 }
 
 func _internal_collectCacheUsageStats(account: Account, peerId: PeerId? = nil, additionalCachePaths: [String] = [], logFilesPath: String? = nil) -> Signal<CacheUsageStatsResult, NoError> {
+    if "".isEmpty {
+        return account.postbox.mediaBox.collectAllResourceUsage()
+        |> mapToSignal { resourceList -> Signal<CacheUsageStatsResult, NoError> in
+            return account.postbox.mediaBox.storageBox.get(ids: resourceList.compactMap { item -> Data? in
+                return item.id?.data(using: .utf8)
+            })
+            |> mapToSignal { entries -> Signal<CacheUsageStatsResult, NoError> in
+                return account.postbox.transaction { transaction -> CacheUsageStatsResult in
+                    var media: [PeerId: [PeerCacheUsageCategory: [MediaId: Int64]]] = [:]
+                    var mediaResourceIds: [MediaId: [MediaResourceId]] = [:]
+                    
+                    media.removeAll()
+                    mediaResourceIds.removeAll()
+                    
+                    let mediaBox = account.postbox.mediaBox
+                    
+                    var totalSize: Int64 = 0
+                    var mediaSize: Int64 = 0
+                    
+                    var processedResourceIds = Set<String>()
+                    
+                    for entry in entries {
+                        let resourceId = MediaResourceId(String(data: entry.id, encoding: .utf8)!)
+                        let resourceSize = mediaBox.resourceUsage(id: resourceId)
+                        if resourceSize != 0 {
+                            totalSize += resourceSize
+                            
+                            for reference in entry.references {
+                                if let message = transaction.getMessage(MessageId(peerId: reference.peerId, namespace: MessageId.Namespace(reference.messageNamespace), id: reference.messageId)) {
+                                    for mediaItem in message.media {
+                                        guard let mediaId = mediaItem.id else {
+                                            continue
+                                        }
+                                        var category: PeerCacheUsageCategory?
+                                        if let _ = mediaItem as? TelegramMediaImage {
+                                            category = .image
+                                        } else if let mediaItem = mediaItem as? TelegramMediaFile {
+                                            if mediaItem.isMusic || mediaItem.isVoice {
+                                                category = .audio
+                                            } else if mediaItem.isVideo {
+                                                category = .video
+                                            } else {
+                                                category = .file
+                                            }
+                                        }
+                                        if let category = category {
+                                            mediaSize += resourceSize
+                                            processedResourceIds.insert(resourceId.stringRepresentation)
+                                            
+                                            media[reference.peerId, default: [:]][category, default: [:]][mediaId, default: 0] += resourceSize
+                                            if let index = mediaResourceIds.index(forKey: mediaId) {
+                                                if !mediaResourceIds[index].value.contains(resourceId) {
+                                                    mediaResourceIds[mediaId]?.append(resourceId)
+                                                }
+                                            } else {
+                                                mediaResourceIds[mediaId] = [resourceId]
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    
+                    var peers: [PeerId: Peer] = [:]
+                    for peerId in media.keys {
+                        if let peer = transaction.getPeer(peerId) {
+                            peers[peer.id] = peer
+                        }
+                    }
+                    
+                    var tempPaths: [String] = []
+                    var tempSize: Int64 = 0
+                    #if os(iOS)
+                        if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: NSTemporaryDirectory()), includingPropertiesForKeys: [.isDirectoryKey, .fileAllocatedSizeKey, .isSymbolicLinkKey]) {
+                            for url in enumerator {
+                                if let url = url as? URL {
+                                    if let isDirectoryValue = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectoryValue {
+                                        tempPaths.append(url.path)
+                                    } else if let fileSizeValue = (try? url.resourceValues(forKeys: Set([.fileAllocatedSizeKey])))?.fileAllocatedSize {
+                                        tempPaths.append(url.path)
+                                        
+                                        if let isSymbolicLinkValue = (try? url.resourceValues(forKeys: Set([.isSymbolicLinkKey])))?.isSymbolicLink, isSymbolicLinkValue {
+                                        } else {
+                                            tempSize += Int64(fileSizeValue)
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    #endif
+                    
+                    var immutableSize: Int64 = 0
+                    if let files = try? FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: account.basePath + "/postbox/db"), includingPropertiesForKeys: [URLResourceKey.fileSizeKey], options: []) {
+                        for url in files {
+                            if let fileSize = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize {
+                                immutableSize += Int64(fileSize)
+                            }
+                        }
+                    }
+                    if let logFilesPath = logFilesPath, let files = try? FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: logFilesPath), includingPropertiesForKeys: [URLResourceKey.fileSizeKey], options: []) {
+                        for url in files {
+                            if let fileSize = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize {
+                                immutableSize += Int64(fileSize)
+                            }
+                        }
+                    }
+                    
+                    for additionalPath in additionalCachePaths {
+                        if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: additionalPath), includingPropertiesForKeys: [.isDirectoryKey, .fileAllocatedSizeKey, .isSymbolicLinkKey]) {
+                            for url in enumerator {
+                                if let url = url as? URL {
+                                    if let isDirectoryValue = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectoryValue {
+                                    } else if let fileSizeValue = (try? url.resourceValues(forKeys: Set([.fileAllocatedSizeKey])))?.fileAllocatedSize {
+                                        tempPaths.append(url.path)
+
+                                        if let isSymbolicLinkValue = (try? url.resourceValues(forKeys: Set([.isSymbolicLinkKey])))?.isSymbolicLink, isSymbolicLinkValue {
+                                        } else {
+                                            tempSize += Int64(fileSizeValue)
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    
+                    var otherSize: Int64 = 0
+                    var otherPaths: [String] = []
+                    
+                    for (id, name, size) in resourceList {
+                        if size == 0 {
+                            continue
+                        }
+                        if let id = id, processedResourceIds.contains(id) {
+                            continue
+                        }
+                        otherSize += size
+                        otherPaths.append(name)
+                    }
+                    
+                    var cacheSize: Int64 = 0
+                    let basePath = account.postbox.mediaBox.basePath
+                    if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: basePath + "/cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
+                        loop: for url in enumerator {
+                            if let url = url as? URL {
+                                if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
+                                    otherPaths.append("cache/" + url.lastPathComponent)
+                                    cacheSize += Int64(value)
+                                }
+                            }
+                        }
+                    }
+                    
+                    func processRecursive(directoryPath: String, subdirectoryPath: String) {
+                        if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: directoryPath), includingPropertiesForKeys: [.fileSizeKey, .isDirectoryKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
+                            loop: for url in enumerator {
+                                if let url = url as? URL {
+                                    if let isDirectory = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectory {
+                                        processRecursive(directoryPath: url.path, subdirectoryPath: subdirectoryPath + "/\(url.lastPathComponent)")
+                                    } else if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
+                                        otherPaths.append("\(subdirectoryPath)/" + url.lastPathComponent)
+                                        cacheSize += Int64(value)
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    
+                    processRecursive(directoryPath: basePath + "/animation-cache", subdirectoryPath: "animation-cache")
+                    
+                    if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: basePath + "/short-cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
+                        loop: for url in enumerator {
+                            if let url = url as? URL {
+                                if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
+                                    otherPaths.append("short-cache/" + url.lastPathComponent)
+                                    cacheSize += Int64(value)
+                                }
+                            }
+                        }
+                    }
+                    
+                    return .result(CacheUsageStats(
+                        media: media,
+                        mediaResourceIds: mediaResourceIds,
+                        peers: peers,
+                        otherSize: otherSize,
+                        otherPaths: otherPaths,
+                        cacheSize: 0,
+                        tempPaths: tempPaths,
+                        tempSize: tempSize,
+                        immutableSize: immutableSize
+                    ))
+                }
+            }
+        }
+    }
+    
     let initialState = CacheUsageStatsState()
     if let peerId = peerId {
         initialState.lowerBound = MessageIndex.lowerBound(peerId: peerId)
@@ -277,7 +474,8 @@ func _internal_collectCacheUsageStats(account: Account, peerId: PeerId? = nil, a
         
         let signal = (fetch |> mapToSignal { mediaByPeer, mediaRefs, updatedLowerBound -> Signal<CacheUsageStatsResult, CollectCacheUsageStatsError> in
             return process(mediaByPeer, mediaRefs, updatedLowerBound)
-        }) |> restart
+        })
+        |> restart
         
         return signal |> `catch` { error in
             switch error {

submodules/TelegramCore/Sources/TelegramEngine/Resources/TelegramEngineResources.swift

@@ -293,6 +293,7 @@ public extension TelegramEngine {
                         preferBackgroundReferenceRevalidation: false,
                         continueInBackground: false
                     ),
+                    location: nil,
                     isRandomAccessAllowed: true
                 ))
                 |> map { result -> EngineMediaResource.Fetch.Result in