ThePain/Swiftgram
1
0
Fork 0
mirror of https://github.com/Swiftgram/Telegram-iOS.git synced 2025-06-16 05:55:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve media index

Browse Source
This commit is contained in:
Ali 2022-12-10 00:23:25 +04:00
parent 683166d411
commit 0bc15c8efa
13 changed files with 691 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Telegram/BUILD
View File

@ -1793,6 +1793,7 @@ plist_fragment(
<key>BGTaskSchedulerPermittedIdentifiers</key>
<array>
<string>{telegram_bundle_id}.refresh</string>
<string>{telegram_bundle_id}.cleanup</string>
</array>
<key>CFBundleAllowMixedLocalizations</key>
<true/>
@ -1893,6 +1894,7 @@ plist_fragment(
<string>location</string>
<string>remote-notification</string>
<string>voip</string>
<string>processing</string>
</array>
<key>UIDeviceFamily</key>
<array>

7
Telegram/NotificationService/Sources/NotificationService.swift
View File

@ -1169,6 +1169,12 @@ private final class NotificationServiceHandler {
parameters: MediaResourceFetchParameters(
tag: nil,
info: resourceFetchInfo(resource: resource),
location: messageId.flatMap { messageId in
return MediaResourceStorageLocation(
peerId: peerId,
messageId: messageId
)
},
isRandomAccessAllowed: true
),
encryptionKey: nil,
@ -1220,6 +1226,7 @@ private final class NotificationServiceHandler {
parameters: MediaResourceFetchParameters(
tag: nil,
info: resourceFetchInfo(resource: resource),
location: nil,
isRandomAccessAllowed: true
),
encryptionKey: nil,

2
submodules/ContextUI/Sources/ContextActionsContainerNode.swift
View File

@ -586,7 +586,7 @@ final class InnerTextSelectionTipContainerNode: ASDisplayNode {
let textRightInset: CGFloat
if let _ = self.iconNode.image {
textRightInset = iconSize.width - 8.0
textRightInset = iconSize.width - 2.0
} else {
textRightInset = 0.0
}

1
submodules/Postbox/BUILD
View File

@ -17,6 +17,7 @@ swift_library(
"//submodules/StringTransliteration:StringTransliteration",
"//submodules/ManagedFile:ManagedFile",
"//submodules/Utils/RangeSet:RangeSet",
"//submodules/CryptoUtils",
],
visibility = [
"//visibility:public",

126
submodules/Postbox/Sources/MediaBox.swift
View File

@ -144,6 +144,8 @@ public final class MediaBox {
private let cacheQueue = Queue()
private let timeBasedCleanup: TimeBasedCleanup
public let storageBox: StorageBox
private let didRemoveResourcesPipe = ValuePipe<Void>()
public var didRemoveResources: Signal<Void, NoError> {
return .single(Void()) |> then(self.didRemoveResourcesPipe.signal())
@ -187,6 +189,10 @@ public final class MediaBox {
public init(basePath: String) {
self.basePath = basePath
self.storageBox = StorageBox(logger: StorageBox.Logger(impl: { string in
postboxLog(string)
}), basePath: basePath + "/storage")
self.timeBasedCleanup = TimeBasedCleanup(generalPaths: [
self.basePath,
self.basePath + "/cache",
@ -204,6 +210,14 @@ public final class MediaBox {
self.timeBasedCleanup.setMaxStoreTimes(general: general, shortLived: shortLived, gigabytesLimit: gigabytesLimit)
}
private func idForFileName(name: String) -> String {
if name.hasSuffix("_partial") {
return String(name[name.startIndex ..< name.index(name.endIndex, offsetBy: -8)])
} else {
return name
}
}
private func fileNameForId(_ id: MediaResourceId) -> String {
return "\(id.stringRepresentation)"
}
@ -580,6 +594,10 @@ public final class MediaBox {
return
}
if let location = parameters?.location {
self.storageBox.add(reference: StorageBox.Reference(peerId: location.peerId, messageNamespace: UInt8(clamping: location.messageId.namespace), messageId: location.messageId.id), to: resource.id.stringRepresentation.data(using: .utf8)!)
}
guard let (fileContext, releaseContext) = self.fileContext(for: resource.id) else {
subscriber.putCompletion()
return
@ -742,6 +760,10 @@ public final class MediaBox {
self.dataQueue.async {
let paths = self.storePathsForId(resource.id)
if let location = parameters?.location {
self.storageBox.add(reference: StorageBox.Reference(peerId: location.peerId, messageNamespace: UInt8(clamping: location.messageId.namespace), messageId: location.messageId.id), to: resource.id.stringRepresentation.data(using: .utf8)!)
}
if let _ = fileSize(paths.complete) {
if implNext {
subscriber.putNext(.local)
@ -1174,6 +1196,17 @@ public final class MediaBox {
}
}
public func resourceUsage(id: MediaResourceId) -> Int64 {
let paths = self.storePathsForId(id)
if let size = fileSize(paths.complete) {
return Int64(size)
} else if let size = fileSize(paths.partial, useTotalFileAllocatedSize: true) {
return Int64(size)
} else {
return 0
}
}
public func collectResourceCacheUsage(_ ids: [MediaResourceId]) -> Signal<[MediaResourceId: Int64], NoError> {
return Signal { subscriber in
self.dataQueue.async {
@ -1194,6 +1227,99 @@ public final class MediaBox {
}
}
public func collectAllResourceUsage() -> Signal<[(id: String?, path: String, size: Int64)], NoError> {
return Signal { subscriber in
self.dataQueue.async {
var result: [(id: String?, path: String, size: Int64)] = []
var fileIds = Set<Data>()
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: self.basePath), includingPropertiesForKeys: [.fileSizeKey, .fileResourceIdentifierKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let fileId = (try? url.resourceValues(forKeys: Set([.fileResourceIdentifierKey])))?.fileResourceIdentifier as? Data {
if fileIds.contains(fileId) {
//paths.append(url.lastPathComponent)
continue loop
}
if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
fileIds.insert(fileId)
result.append((id: self.idForFileName(name: url.lastPathComponent), path: url.lastPathComponent, size: Int64(value)))
//paths.append(url.lastPathComponent)
}
}
}
}
}
/*var cacheResult: Int64 = 0
var excludePrefixes = Set<String>()
for id in excludeIds {
let cachedRepresentationPrefix = self.fileNameForId(id)
excludePrefixes.insert(cachedRepresentationPrefix)
}
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: self.basePath + "/cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let prefix = url.lastPathComponent.components(separatedBy: ":").first, excludePrefixes.contains(prefix) {
continue loop
}
if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
paths.append("cache/" + url.lastPathComponent)
cacheResult += Int64(value)
}
}
}
}
func processRecursive(directoryPath: String, subdirectoryPath: String) {
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: directoryPath), includingPropertiesForKeys: [.fileSizeKey, .isDirectoryKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let prefix = url.lastPathComponent.components(separatedBy: ":").first, excludePrefixes.contains(prefix) {
continue loop
}
if let isDirectory = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectory {
processRecursive(directoryPath: url.path, subdirectoryPath: subdirectoryPath + "/\(url.lastPathComponent)")
} else if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
paths.append("\(subdirectoryPath)/" + url.lastPathComponent)
cacheResult += Int64(value)
}
}
}
}
}
processRecursive(directoryPath: self.basePath + "/animation-cache", subdirectoryPath: "animation-cache")
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: self.basePath + "/short-cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let prefix = url.lastPathComponent.components(separatedBy: ":").first, excludePrefixes.contains(prefix) {
continue loop
}
if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
paths.append("short-cache/" + url.lastPathComponent)
cacheResult += Int64(value)
}
}
}
}*/
subscriber.putNext(result)
subscriber.putCompletion()
}
return EmptyDisposable
}
}
public func collectOtherResourceUsage(excludeIds: Set<MediaResourceId>, combinedExcludeIds: Set<MediaResourceId>) -> Signal<(Int64, [String], Int64), NoError> {
return Signal { subscriber in
self.dataQueue.async {

14
submodules/Postbox/Sources/MediaResource.swift
View File

@ -39,14 +39,26 @@ public protocol MediaResourceFetchTag {
public protocol MediaResourceFetchInfo {
}
public final class MediaResourceStorageLocation {
public let peerId: PeerId
public let messageId: MessageId
public init(peerId: PeerId, messageId: MessageId) {
self.peerId = peerId
self.messageId = messageId
}
}
public struct MediaResourceFetchParameters {
public let tag: MediaResourceFetchTag?
public let info: MediaResourceFetchInfo?
public let location: MediaResourceStorageLocation?
public let isRandomAccessAllowed: Bool
public init(tag: MediaResourceFetchTag?, info: MediaResourceFetchInfo?, isRandomAccessAllowed: Bool) {
public init(tag: MediaResourceFetchTag?, info: MediaResourceFetchInfo?, location: MediaResourceStorageLocation?, isRandomAccessAllowed: Bool) {
self.tag = tag
self.info = info
self.location = location
self.isRandomAccessAllowed = isRandomAccessAllowed
}
}

89
submodules/Postbox/Sources/SqliteValueBox.swift
View File

@ -188,6 +188,8 @@ public final class SqliteValueBox: ValueBox {
private var updateStatements: [Int32 : SqlitePreparedStatement] = [:]
private var insertOrReplacePrimaryKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
private var insertOrReplaceIndexKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
private var insertOrIgnorePrimaryKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
private var insertOrIgnoreIndexKeyStatements: [Int32 : SqlitePreparedStatement] = [:]
private var deleteStatements: [Int32 : SqlitePreparedStatement] = [:]
private var moveStatements: [Int32 : SqlitePreparedStatement] = [:]
private var copyStatements: [TablePairKey : SqlitePreparedStatement] = [:]
@ -1220,6 +1222,59 @@ public final class SqliteValueBox: ValueBox {
return resultStatement
}
private func insertOrIgnoreStatement(_ table: SqliteValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) -> SqlitePreparedStatement {
precondition(self.queue.isCurrent())
checkTableKey(table.table, key)
let resultStatement: SqlitePreparedStatement
if table.table.keyType == .int64 || table.hasPrimaryKey {
if let statement = self.insertOrIgnorePrimaryKeyStatements[table.table.id] {
resultStatement = statement
} else {
var statement: OpaquePointer? = nil
let status = sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table.table.id) (key, value) VALUES(?, ?) ON CONFLICT(key) DO NOTHING", -1, &statement, nil)
if status != SQLITE_OK {
let errorText = self.database.currentError() ?? "Unknown error"
preconditionFailure(errorText)
}
let preparedStatement = SqlitePreparedStatement(statement: statement)
self.insertOrIgnorePrimaryKeyStatements[table.table.id] = preparedStatement
resultStatement = preparedStatement
}
} else {
if let statement = self.insertOrIgnoreIndexKeyStatements[table.table.id] {
resultStatement = statement
} else {
var statement: OpaquePointer? = nil
let status = sqlite3_prepare_v2(self.database.handle, "INSERT INTO t\(table.table.id) (key, value) VALUES(?, ?)", -1, &statement, nil)
if status != SQLITE_OK {
let errorText = self.database.currentError() ?? "Unknown error"
preconditionFailure(errorText)
}
let preparedStatement = SqlitePreparedStatement(statement: statement)
self.insertOrIgnorePrimaryKeyStatements[table.table.id] = preparedStatement
resultStatement = preparedStatement
}
}
resultStatement.reset()
switch table.table.keyType {
case .binary:
resultStatement.bind(1, data: key.memory, length: key.length)
case .int64:
resultStatement.bind(1, number: key.getInt64(0))
}
if value.length == 0 {
resultStatement.bindNull(2)
} else {
resultStatement.bind(2, data: value.memory, length: value.length)
}
return resultStatement
}
private func deleteStatement(_ table: ValueBoxTable, key: ValueBoxKey) -> SqlitePreparedStatement {
precondition(self.queue.isCurrent())
checkTableKey(table, key)
@ -1847,6 +1902,30 @@ public final class SqliteValueBox: ValueBox {
}
}
public func setOrIgnore(_ table: ValueBoxTable, key: ValueBoxKey, value: MemoryBuffer) {
precondition(self.queue.isCurrent())
let sqliteTable = self.checkTable(table)
if sqliteTable.hasPrimaryKey {
let statement = self.insertOrIgnoreStatement(sqliteTable, key: key, value: value)
while statement.step(handle: self.database.handle, pathToRemoveOnError: self.removeDatabaseOnError ? self.databasePath : nil) {
}
statement.reset()
} else {
if self.exists(table, key: key) {
let statement = self.updateStatement(table, key: key, value: value)
while statement.step(handle: self.database.handle, pathToRemoveOnError: self.removeDatabaseOnError ? self.databasePath : nil) {
}
statement.reset()
} else {
let statement = self.insertOrReplaceStatement(sqliteTable, key: key, value: value)
while statement.step(handle: self.database.handle, pathToRemoveOnError: self.removeDatabaseOnError ? self.databasePath : nil) {
}
statement.reset()
}
}
}
public func remove(_ table: ValueBoxTable, key: ValueBoxKey, secure: Bool) {
precondition(self.queue.isCurrent())
if let _ = self.tables[table.id] {
@ -2108,6 +2187,16 @@ public final class SqliteValueBox: ValueBox {
}
self.insertOrReplacePrimaryKeyStatements.removeAll()
for (_, statement) in self.insertOrIgnoreIndexKeyStatements {
statement.destroy()
}
self.insertOrIgnoreIndexKeyStatements.removeAll()
for (_, statement) in self.insertOrIgnorePrimaryKeyStatements {
statement.destroy()
}
self.insertOrIgnorePrimaryKeyStatements.removeAll()
for (_, statement) in self.deleteStatements {
statement.destroy()
}

205
submodules/Postbox/Sources/StorageBox/StorageBox.swift Normal file
View File

@ -0,0 +1,205 @@
import Foundation
import SwiftSignalKit
import CryptoUtils
public struct HashId: Hashable {
public let data: Data
public init(data: Data) {
precondition(data.count == 16)
self.data = data
}
}
private func md5Hash(_ data: Data) -> HashId {
let hashData = data.withUnsafeBytes { bytes -> Data in
return CryptoMD5(bytes.baseAddress!, Int32(bytes.count))
}
return HashId(data: hashData)
}
public final class StorageBox {
public struct Reference {
public var peerId: PeerId
public var messageNamespace: UInt8
public var messageId: Int32
public init(peerId: PeerId, messageNamespace: UInt8, messageId: Int32) {
self.peerId = peerId
self.messageNamespace = messageNamespace
self.messageId = messageId
}
}
public final class Entry {
public let id: Data
public let references: [Reference]
init(id: Data, references: [Reference]) {
self.id = id
self.references = references
}
}
public final class Logger {
private let impl: (String) -> Void
public init(impl: @escaping (String) -> Void) {
self.impl = impl
}
func log(_ string: @autoclosure () -> String) {
self.impl(string())
}
}
private final class Impl {
let queue: Queue
let logger: StorageBox.Logger
let basePath: String
let valueBox: SqliteValueBox
let hashIdToIdTable: ValueBoxTable
let idToReferenceTable: ValueBoxTable
init(queue: Queue, logger: StorageBox.Logger, basePath: String) {
self.queue = queue
self.logger = logger
self.basePath = basePath
let databasePath = self.basePath + "/db"
let _ = try? FileManager.default.createDirectory(atPath: databasePath, withIntermediateDirectories: true)
var valueBox = SqliteValueBox(basePath: databasePath, queue: queue, isTemporary: false, isReadOnly: false, useCaches: true, removeDatabaseOnError: true, encryptionParameters: nil, upgradeProgress: { _ in })
if valueBox == nil {
let _ = try? FileManager.default.removeItem(atPath: databasePath)
valueBox = SqliteValueBox(basePath: databasePath, queue: queue, isTemporary: false, isReadOnly: false, useCaches: true, removeDatabaseOnError: true, encryptionParameters: nil, upgradeProgress: { _ in })
}
guard let valueBox else {
preconditionFailure("Could not open database")
}
self.valueBox = valueBox
self.hashIdToIdTable = ValueBoxTable(id: 5, keyType: .binary, compactValuesOnCreation: true)
self.idToReferenceTable = ValueBoxTable(id: 6, keyType: .binary, compactValuesOnCreation: true)
}
func add(reference: Reference, to id: Data) {
self.valueBox.begin()
let hashId = md5Hash(id)
let mainKey = ValueBoxKey(length: hashId.data.count)
self.valueBox.setOrIgnore(self.hashIdToIdTable, key: mainKey, value: MemoryBuffer(data: id))
let idKey = ValueBoxKey(length: hashId.data.count + 8 + 1 + 4)
idKey.setData(0, value: hashId.data)
idKey.setInt64(hashId.data.count, value: reference.peerId.toInt64())
idKey.setUInt8(hashId.data.count + 8, value: reference.messageNamespace)
idKey.setInt32(hashId.data.count + 8 + 1, value: reference.messageId)
self.valueBox.setOrIgnore(self.idToReferenceTable, key: idKey, value: MemoryBuffer())
self.valueBox.commit()
}
func all() -> [Entry] {
var result: [Entry] = []
self.valueBox.begin()
var currentId: Data?
var currentReferences: [Reference] = []
self.valueBox.scan(self.idToReferenceTable, keys: { key in
let id = key.getData(0, length: 16)
let peerId = PeerId(key.getInt64(16))
let messageNamespace: UInt8 = key.getUInt8(16 + 8)
let messageId = key.getInt32(16 + 8 + 1)
let reference = Reference(peerId: peerId, messageNamespace: messageNamespace, messageId: messageId)
if currentId == id {
currentReferences.append(reference)
} else {
if let currentId = currentId, !currentReferences.isEmpty {
result.append(StorageBox.Entry(id: currentId, references: currentReferences))
currentReferences.removeAll(keepingCapacity: true)
}
currentId = id
currentReferences.append(reference)
}
return true
})
self.valueBox.commit()
return result
}
func get(ids: [Data]) -> [Entry] {
var result: [Entry] = []
self.valueBox.begin()
let idKey = ValueBoxKey(length: 16)
for id in ids {
let hashId = md5Hash(id)
idKey.setData(0, value: hashId.data)
var currentReferences: [Reference] = []
self.valueBox.range(self.idToReferenceTable, start: idKey, end: idKey.successor, keys: { key in
let peerId = PeerId(key.getInt64(16))
let messageNamespace: UInt8 = key.getUInt8(16 + 8)
let messageId = key.getInt32(16 + 8 + 1)
let reference = Reference(peerId: peerId, messageNamespace: messageNamespace, messageId: messageId)
currentReferences.append(reference)
return true
}, limit: 0)
if !currentReferences.isEmpty {
result.append(StorageBox.Entry(id: id, references: currentReferences))
}
}
self.valueBox.commit()
return result
}
}
private let queue: Queue
private let impl: QueueLocalObject<Impl>
public init(queue: Queue = Queue(name: "StorageBox"), logger: StorageBox.Logger, basePath: String) {
self.queue = queue
self.impl = QueueLocalObject(queue: queue, generate: {
return Impl(queue: queue, logger: logger, basePath: basePath)
})
}
public func add(reference: Reference, to id: Data) {
self.impl.with { impl in
impl.add(reference: reference, to: id)
}
}
public func all() -> Signal<[Entry], NoError> {
return self.impl.signalWith { impl, subscriber in
subscriber.putNext(impl.all())
subscriber.putCompletion()
return EmptyDisposable
}
}
public func get(ids: [Data]) -> Signal<[Entry], NoError> {
return self.impl.signalWith { impl, subscriber in
subscriber.putNext(impl.get(ids: ids))
subscriber.putCompletion()
return EmptyDisposable
}
}
}

9
submodules/Postbox/Sources/ValueBoxKey.swift
View File

@ -83,6 +83,15 @@ public struct ValueBoxKey: Equatable, Hashable, CustomStringConvertible, Compara
memcpy(self.memory + offset, &varValue, 2)
}
public func getData(_ offset: Int, length: Int) -> Data {
assert(offset >= 0 && offset + length <= self.length)
var value = Data(count: length)
let _ = value.withUnsafeMutableBytes { bytes in
memcpy(bytes.baseAddress!.assumingMemoryBound(to: UInt8.self), self.memory + offset, length)
}
return value
}
public func getInt32(_ offset: Int) -> Int32 {
assert(offset >= 0 && offset + 4 <= self.length)
var value: Int32 = 0

34
submodules/TelegramCore/Sources/Network/FetchedMediaResource.swift
View File

@ -29,6 +29,26 @@ public func fetchedMediaResource(mediaBox: MediaBox, reference: MediaResourceRef
return fetchedMediaResource(mediaBox: mediaBox, reference: reference, ranges: range.flatMap({ [$0] }), statsCategory: statsCategory, reportResultStatus: reportResultStatus, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground)
}
public extension MediaResourceStorageLocation {
convenience init?(reference: MediaResourceReference) {
switch reference {
case let .media(media, _):
switch media {
case let .message(message, _):
if let id = message.id {
self.init(peerId: id.peerId, messageId: id)
} else {
return nil
}
default:
return nil
}
default:
return nil
}
}
}
public func fetchedMediaResource(mediaBox: MediaBox, reference: MediaResourceReference, ranges: [(Range<Int64>, MediaBoxFetchPriority)]?, statsCategory: MediaResourceStatsCategory = .generic, reportResultStatus: Bool = false, preferBackgroundReferenceRevalidation: Bool = false, continueInBackground: Bool = false) -> Signal<FetchResourceSourceType, FetchResourceError> {
var isRandomAccessAllowed = true
switch reference {
@ -44,14 +64,24 @@ public func fetchedMediaResource(mediaBox: MediaBox, reference: MediaResourceRef
if let ranges = ranges {
let signals = ranges.map { (range, priority) -> Signal<Void, FetchResourceError> in
return mediaBox.fetchedResourceData(reference.resource, in: range, priority: priority, parameters: MediaResourceFetchParameters(tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory), info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground), isRandomAccessAllowed: isRandomAccessAllowed))
return mediaBox.fetchedResourceData(reference.resource, in: range, priority: priority, parameters: MediaResourceFetchParameters(
tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory),
info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground),
location: MediaResourceStorageLocation(reference: reference),
isRandomAccessAllowed: isRandomAccessAllowed
))
}
return combineLatest(signals)
|> ignoreValues
|> map { _ -> FetchResourceSourceType in }
|> then(.single(.local))
} else {
return mediaBox.fetchedResource(reference.resource, parameters: MediaResourceFetchParameters(tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory), info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground), isRandomAccessAllowed: isRandomAccessAllowed), implNext: reportResultStatus)
return mediaBox.fetchedResource(reference.resource, parameters: MediaResourceFetchParameters(
tag: TelegramMediaResourceFetchTag(statsCategory: statsCategory),
info: TelegramCloudMediaResourceFetchInfo(reference: reference, preferBackgroundReferenceRevalidation: preferBackgroundReferenceRevalidation, continueInBackground: continueInBackground),
location: MediaResourceStorageLocation(reference: reference),
isRandomAccessAllowed: isRandomAccessAllowed
), implNext: reportResultStatus)
}
}

7
submodules/TelegramCore/Sources/State/Fetch.swift
View File

@ -75,7 +75,12 @@ func fetchResource(account: Account, resource: MediaResource, intervals: Signal<
return .fail(.generic)
}
return .single(.dataPart(resourceOffset: 0, data: Data(), range: 0 ..< 0, complete: false))
|> then(fetchCloudMediaLocation(account: account, resource: cloudResource, datacenterId: cloudResource.datacenterId, size: resource.size == 0 ? nil : resource.size, intervals: intervals, parameters: MediaResourceFetchParameters(tag: nil, info: TelegramCloudMediaResourceFetchInfo(reference: .standalone(resource: file.file.resource), preferBackgroundReferenceRevalidation: false, continueInBackground: false), isRandomAccessAllowed: true)))
|> then(fetchCloudMediaLocation(account: account, resource: cloudResource, datacenterId: cloudResource.datacenterId, size: resource.size == 0 ? nil : resource.size, intervals: intervals, parameters: MediaResourceFetchParameters(
tag: nil,
info: TelegramCloudMediaResourceFetchInfo(reference: .standalone(resource: file.file.resource), preferBackgroundReferenceRevalidation: false, continueInBackground: false),
location: nil,
isRandomAccessAllowed: true
)))
}
}
return nil

200
submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift
View File

@ -53,6 +53,203 @@ private final class CacheUsageStatsState {
}
func _internal_collectCacheUsageStats(account: Account, peerId: PeerId? = nil, additionalCachePaths: [String] = [], logFilesPath: String? = nil) -> Signal<CacheUsageStatsResult, NoError> {
if "".isEmpty {
return account.postbox.mediaBox.collectAllResourceUsage()
|> mapToSignal { resourceList -> Signal<CacheUsageStatsResult, NoError> in
return account.postbox.mediaBox.storageBox.get(ids: resourceList.compactMap { item -> Data? in
return item.id?.data(using: .utf8)
})
|> mapToSignal { entries -> Signal<CacheUsageStatsResult, NoError> in
return account.postbox.transaction { transaction -> CacheUsageStatsResult in
var media: [PeerId: [PeerCacheUsageCategory: [MediaId: Int64]]] = [:]
var mediaResourceIds: [MediaId: [MediaResourceId]] = [:]
media.removeAll()
mediaResourceIds.removeAll()
let mediaBox = account.postbox.mediaBox
var totalSize: Int64 = 0
var mediaSize: Int64 = 0
var processedResourceIds = Set<String>()
for entry in entries {
let resourceId = MediaResourceId(String(data: entry.id, encoding: .utf8)!)
let resourceSize = mediaBox.resourceUsage(id: resourceId)
if resourceSize != 0 {
totalSize += resourceSize
for reference in entry.references {
if let message = transaction.getMessage(MessageId(peerId: reference.peerId, namespace: MessageId.Namespace(reference.messageNamespace), id: reference.messageId)) {
for mediaItem in message.media {
guard let mediaId = mediaItem.id else {
continue
}
var category: PeerCacheUsageCategory?
if let _ = mediaItem as? TelegramMediaImage {
category = .image
} else if let mediaItem = mediaItem as? TelegramMediaFile {
if mediaItem.isMusic || mediaItem.isVoice {
category = .audio
} else if mediaItem.isVideo {
category = .video
} else {
category = .file
}
}
if let category = category {
mediaSize += resourceSize
processedResourceIds.insert(resourceId.stringRepresentation)
media[reference.peerId, default: [:]][category, default: [:]][mediaId, default: 0] += resourceSize
if let index = mediaResourceIds.index(forKey: mediaId) {
if !mediaResourceIds[index].value.contains(resourceId) {
mediaResourceIds[mediaId]?.append(resourceId)
}
} else {
mediaResourceIds[mediaId] = [resourceId]
}
}
}
}
}
}
}
var peers: [PeerId: Peer] = [:]
for peerId in media.keys {
if let peer = transaction.getPeer(peerId) {
peers[peer.id] = peer
}
}
var tempPaths: [String] = []
var tempSize: Int64 = 0
#if os(iOS)
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: NSTemporaryDirectory()), includingPropertiesForKeys: [.isDirectoryKey, .fileAllocatedSizeKey, .isSymbolicLinkKey]) {
for url in enumerator {
if let url = url as? URL {
if let isDirectoryValue = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectoryValue {
tempPaths.append(url.path)
} else if let fileSizeValue = (try? url.resourceValues(forKeys: Set([.fileAllocatedSizeKey])))?.fileAllocatedSize {
tempPaths.append(url.path)
if let isSymbolicLinkValue = (try? url.resourceValues(forKeys: Set([.isSymbolicLinkKey])))?.isSymbolicLink, isSymbolicLinkValue {
} else {
tempSize += Int64(fileSizeValue)
}
}
}
}
}
#endif
var immutableSize: Int64 = 0
if let files = try? FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: account.basePath + "/postbox/db"), includingPropertiesForKeys: [URLResourceKey.fileSizeKey], options: []) {
for url in files {
if let fileSize = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize {
immutableSize += Int64(fileSize)
}
}
}
if let logFilesPath = logFilesPath, let files = try? FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: logFilesPath), includingPropertiesForKeys: [URLResourceKey.fileSizeKey], options: []) {
for url in files {
if let fileSize = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize {
immutableSize += Int64(fileSize)
}
}
}
for additionalPath in additionalCachePaths {
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: additionalPath), includingPropertiesForKeys: [.isDirectoryKey, .fileAllocatedSizeKey, .isSymbolicLinkKey]) {
for url in enumerator {
if let url = url as? URL {
if let isDirectoryValue = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectoryValue {
} else if let fileSizeValue = (try? url.resourceValues(forKeys: Set([.fileAllocatedSizeKey])))?.fileAllocatedSize {
tempPaths.append(url.path)
if let isSymbolicLinkValue = (try? url.resourceValues(forKeys: Set([.isSymbolicLinkKey])))?.isSymbolicLink, isSymbolicLinkValue {
} else {
tempSize += Int64(fileSizeValue)
}
}
}
}
}
}
var otherSize: Int64 = 0
var otherPaths: [String] = []
for (id, name, size) in resourceList {
if size == 0 {
continue
}
if let id = id, processedResourceIds.contains(id) {
continue
}
otherSize += size
otherPaths.append(name)
}
var cacheSize: Int64 = 0
let basePath = account.postbox.mediaBox.basePath
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: basePath + "/cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
otherPaths.append("cache/" + url.lastPathComponent)
cacheSize += Int64(value)
}
}
}
}
func processRecursive(directoryPath: String, subdirectoryPath: String) {
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: directoryPath), includingPropertiesForKeys: [.fileSizeKey, .isDirectoryKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let isDirectory = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectory {
processRecursive(directoryPath: url.path, subdirectoryPath: subdirectoryPath + "/\(url.lastPathComponent)")
} else if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
otherPaths.append("\(subdirectoryPath)/" + url.lastPathComponent)
cacheSize += Int64(value)
}
}
}
}
}
processRecursive(directoryPath: basePath + "/animation-cache", subdirectoryPath: "animation-cache")
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: basePath + "/short-cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
otherPaths.append("short-cache/" + url.lastPathComponent)
cacheSize += Int64(value)
}
}
}
}
return .result(CacheUsageStats(
media: media,
mediaResourceIds: mediaResourceIds,
peers: peers,
otherSize: otherSize,
otherPaths: otherPaths,
cacheSize: 0,
tempPaths: tempPaths,
tempSize: tempSize,
immutableSize: immutableSize
))
}
}
}
}
let initialState = CacheUsageStatsState()
if let peerId = peerId {
initialState.lowerBound = MessageIndex.lowerBound(peerId: peerId)
@ -277,7 +474,8 @@ func _internal_collectCacheUsageStats(account: Account, peerId: PeerId? = nil, a
let signal = (fetch |> mapToSignal { mediaByPeer, mediaRefs, updatedLowerBound -> Signal<CacheUsageStatsResult, CollectCacheUsageStatsError> in
return process(mediaByPeer, mediaRefs, updatedLowerBound)
}) |> restart
})
|> restart
return signal |> `catch` { error in
switch error {

1
submodules/TelegramCore/Sources/TelegramEngine/Resources/TelegramEngineResources.swift
View File

@ -293,6 +293,7 @@ public extension TelegramEngine {
preferBackgroundReferenceRevalidation: false,
continueInBackground: false
),
location: nil,
isRandomAccessAllowed: true
))
|> map { result -> EngineMediaResource.Fetch.Result in