From c714d23730bc16582cd4e44757a0a428cedde811 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Sun, 25 Dec 2022 00:14:38 +0400 Subject: [PATCH] Storage management improvements --- .../Sources/SolidRoundedButtonComponent.swift | 1 + .../PublicHeaders/MtProtoKit/MTProto.h | 8 --- submodules/Postbox/Sources/MediaBox.swift | 37 +++++++++++- .../Sources/StorageBox/StorageBox.swift | 47 +++++++++++++++ .../Postbox/Sources/TimeBasedCleanup.swift | 58 +++++++++---------- submodules/TelegramCore/BUILD | 1 + .../Resources/CollectCacheUsageStats.swift | 1 + .../StorageFileListPanelComponent.swift | 2 +- .../StoragePeerListPanelComponent.swift | 2 +- .../Sources/StorageUsageScreen.swift | 15 +++-- ...geUsageScreenSelectionPanelComponent.swift | 2 +- submodules/Utils/DarwinDirStat/BUILD | 21 +++++++ submodules/Utils/DarwinDirStat/Package.swift | 32 ++++++++++ .../DarwinDirStat/DarwinDirStat.h | 14 +++++ .../DarwinDirStat/Sources/DarwinDirStat.m | 2 + 15 files changed, 193 insertions(+), 50 deletions(-) create mode 100644 submodules/Utils/DarwinDirStat/BUILD create mode 100644 submodules/Utils/DarwinDirStat/Package.swift create mode 100644 submodules/Utils/DarwinDirStat/PublicHeaders/DarwinDirStat/DarwinDirStat.h create mode 100644 submodules/Utils/DarwinDirStat/Sources/DarwinDirStat.m diff --git a/submodules/Components/SolidRoundedButtonComponent/Sources/SolidRoundedButtonComponent.swift b/submodules/Components/SolidRoundedButtonComponent/Sources/SolidRoundedButtonComponent.swift index 247a6d276f..bb83afb081 100644 --- a/submodules/Components/SolidRoundedButtonComponent/Sources/SolidRoundedButtonComponent.swift +++ b/submodules/Components/SolidRoundedButtonComponent/Sources/SolidRoundedButtonComponent.swift @@ -148,6 +148,7 @@ public final class SolidRoundedButtonComponent: Component { button.gloss = component.gloss button.isEnabled = component.isEnabled + button.isUserInteractionEnabled = component.isEnabled button.updateTheme(component.theme) let height = button.updateLayout(width: availableSize.width, transition: .immediate) diff --git a/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTProto.h b/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTProto.h index b8bcd8f79c..990b94fba6 100644 --- a/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTProto.h +++ b/submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTProto.h @@ -77,11 +77,3 @@ + (NSData *)_manuallyEncryptedMessage:(NSData *)preparedData messageId:(int64_t)messageId authKey:(MTDatacenterAuthKey *)authKey; @end - -//#define DIRSTAT_FAST_ONLY 0x1 -struct darwin_dirstat { - off_t total_size; - uint64_t descendants; -}; - -int dirstat_np(const char *path, int flags, struct darwin_dirstat *ds, size_t dirstat_size); diff --git a/submodules/Postbox/Sources/MediaBox.swift b/submodules/Postbox/Sources/MediaBox.swift index 403858aad5..8068698b12 100644 --- a/submodules/Postbox/Sources/MediaBox.swift +++ b/submodules/Postbox/Sources/MediaBox.swift @@ -211,7 +211,7 @@ public final class MediaBox { self.timeBasedCleanup.setMaxStoreTimes(general: general, shortLived: shortLived, gigabytesLimit: gigabytesLimit) } - private static func idForFileName(name: String) -> String { + public static func idForFileName(name: String) -> String { if name.hasSuffix("_partial.meta") { return String(name[name.startIndex ..< name.index(name.endIndex, offsetBy: -13)]) } else if name.hasSuffix("_partial") { @@ -1291,6 +1291,39 @@ public final class MediaBox { let scanContext = ScanFilesContext(path: basePath) + func processStale(nextId: Data?) { + let _ = (storageBox.enumerateItems(startingWith: nextId, limit: 1000) + |> deliverOn(processQueue)).start(next: { ids, realNextId in + var staleIds: [Data] = [] + + for id in ids { + if let name = String(data: id, encoding: .utf8) { + if self.resourceUsage(id: MediaResourceId(name)) == 0 { + staleIds.append(id) + } + } else { + staleIds.append(id) + } + } + + if !staleIds.isEmpty { + storageBox.remove(ids: staleIds) + } + + if realNextId == nil { + completion() + } else { + if lowImpact { + processQueue.after(0.4, { + processStale(nextId: realNextId) + }) + } else { + processStale(nextId: realNextId) + } + } + }) + } + func processNext() { processQueue.async { if isCancelled { @@ -1299,7 +1332,7 @@ public final class MediaBox { let results = scanContext.nextBatch(count: 32000) if results.isEmpty { - completion() + processStale(nextId: nil) return } diff --git a/submodules/Postbox/Sources/StorageBox/StorageBox.swift b/submodules/Postbox/Sources/StorageBox/StorageBox.swift index 3bff357cdf..68b9ec89cd 100644 --- a/submodules/Postbox/Sources/StorageBox/StorageBox.swift +++ b/submodules/Postbox/Sources/StorageBox/StorageBox.swift @@ -577,6 +577,44 @@ public final class StorageBox { return result } + func enumerateItems(startingWith startId: Data?, limit: Int) -> (ids: [Data], nextStartId: Data?) { + self.valueBox.begin() + + let startKey: ValueBoxKey + if let startId = startId, startId.count == 16 { + startKey = ValueBoxKey(length: 16) + startKey.setData(0, value: startId) + } else { + startKey = ValueBoxKey(length: 1) + startKey.setUInt8(0, value: 0) + } + + let endKey = ValueBoxKey(length: 16) + for i in 0 ..< 16 { + endKey.setUInt8(i, value: 0xff) + } + + var ids: [Data] = [] + var nextKey: ValueBoxKey? + self.valueBox.range(self.hashIdToInfoTable, start: startKey, end: endKey, values: { key, value in + nextKey = key + + let info = ItemInfo(buffer: value) + ids.append(info.id) + + return true + }, limit: limit) + + self.valueBox.commit() + + var nextId = nextKey?.getData(0, length: 16) + if nextId == startId { + nextId = nil + } + + return (ids, nextId) + } + func all() -> [Entry] { var result: [Entry] = [] @@ -912,4 +950,13 @@ public final class StorageBox { impl.reset() } } + + public func enumerateItems(startingWith startId: Data?, limit: Int) -> Signal<(ids: [Data], nextStartId: Data?), NoError> { + return self.impl.signalWith { impl, subscriber in + subscriber.putNext(impl.enumerateItems(startingWith: startId, limit: limit)) + subscriber.putCompletion() + + return EmptyDisposable + } + } } diff --git a/submodules/Postbox/Sources/TimeBasedCleanup.swift b/submodules/Postbox/Sources/TimeBasedCleanup.swift index 9c5fc96258..0436501db8 100644 --- a/submodules/Postbox/Sources/TimeBasedCleanup.swift +++ b/submodules/Postbox/Sources/TimeBasedCleanup.swift @@ -39,32 +39,6 @@ public func printOpenFiles() { } } -/* - +(void) lsof - { - int flags; - int fd; - char buf[MAXPATHLEN+1] ; - int n = 1 ; - - for (fd = 0; fd < (int) FD_SETSIZE; fd++) { - errno = 0; - flags = fcntl(fd, F_GETFD, 0); - if (flags == -1 && errno) { - if (errno != EBADF) { - return ; - } - else - continue; - } - fcntl(fd , F_GETPATH, buf ) ; - NSLog( @"File Descriptor %d number %d in use for: %s",fd,n , buf ) ; - ++n ; - } - } - - */ - private func scanFiles(at path: String, olderThan minTimestamp: Int32, inodes: inout [InodeInfo]) -> ScanFilesResult { var result = ScanFilesResult() @@ -113,7 +87,7 @@ private func scanFiles(at path: String, olderThan minTimestamp: Int32, inodes: i return result } -private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UInt64) { +private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UInt64, mainStoragePath: String, storageBox: StorageBox) { var removedSize: UInt64 = 0 inodes.sort(by: { lhs, rhs in @@ -139,7 +113,10 @@ private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UI free(pathBuffer) } + var unlinkedResourceIds: [Data] = [] + for path in paths { + let isMainPath = path == mainStoragePath if let dp = opendir(path) { while true { guard let dirp = readdir(dp) else { @@ -162,6 +139,17 @@ private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UI var value = stat() if stat(pathBuffer, &value) == 0 { if inodesToDelete.contains(value.st_ino) { + if isMainPath { + let nameLength = strnlen(&dirp.pointee.d_name.0, 1024) + let nameData = Data(bytesNoCopy: &dirp.pointee.d_name.0, count: Int(nameLength), deallocator: .none) + withExtendedLifetime(nameData, { + if let fileName = String(data: nameData, encoding: .utf8) { + if let idData = MediaBox.idForFileName(name: fileName).data(using: .utf8) { + unlinkedResourceIds.append(idData) + } + } + }) + } unlink(pathBuffer) } } @@ -169,6 +157,10 @@ private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UI closedir(dp) } } + + if !unlinkedResourceIds.isEmpty { + storageBox.remove(ids: unlinkedResourceIds) + } } private final class TimeBasedCleanupImpl { @@ -226,6 +218,7 @@ private final class TimeBasedCleanupImpl { let generalPaths = self.generalPaths let totalSizeBasedPath = self.totalSizeBasedPath let shortLivedPaths = self.shortLivedPaths + let storageBox = self.storageBox let scanOnce = Signal { subscriber in DispatchQueue.global(qos: .background).async { var removedShortLivedCount: Int = 0 @@ -238,7 +231,12 @@ private final class TimeBasedCleanupImpl { var paths: [String] = [] let timestamp = Int32(Date().timeIntervalSince1970) + + /*#if DEBUG + let bytesLimit: UInt64 = 10 * 1024 * 1024 + #else*/ let bytesLimit = UInt64(gigabytesLimit) * 1024 * 1024 * 1024 + //#endif let oldestShortLivedTimestamp = timestamp - shortLived let oldestGeneralTimestamp = timestamp - general @@ -270,13 +268,9 @@ private final class TimeBasedCleanupImpl { } if totalLimitSize > bytesLimit { - mapFiles(paths: paths, inodes: &inodes, removeSize: totalLimitSize - bytesLimit) + mapFiles(paths: paths, inodes: &inodes, removeSize: totalLimitSize - bytesLimit, mainStoragePath: totalSizeBasedPath, storageBox: storageBox) } - #if DEBUG - //printOpenFiles() - #endif - if removedShortLivedCount != 0 || removedGeneralCount != 0 || removedGeneralLimitCount != 0 { postboxLog("[TimeBasedCleanup] \(CFAbsoluteTimeGetCurrent() - startTime) s removed \(removedShortLivedCount) short-lived files, \(removedGeneralCount) general files, \(removedGeneralLimitCount) limit files") } diff --git a/submodules/TelegramCore/BUILD b/submodules/TelegramCore/BUILD index 7f444963cd..5a883c9356 100644 --- a/submodules/TelegramCore/BUILD +++ b/submodules/TelegramCore/BUILD @@ -47,6 +47,7 @@ swift_library( "//submodules/Reachability:Reachability", "//submodules/ManagedFile:ManagedFile", "//submodules/Utils/RangeSet:RangeSet", + "//submodules/Utils/DarwinDirStat", ], visibility = [ "//visibility:public", diff --git a/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift b/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift index 719679c2a8..a871b81bcc 100644 --- a/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift +++ b/submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift @@ -2,6 +2,7 @@ import Foundation import Postbox import SwiftSignalKit import MtProtoKit +import DarwinDirStat public enum PeerCacheUsageCategory: Int32 { case image = 0 diff --git a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageFileListPanelComponent.swift b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageFileListPanelComponent.swift index 6ec155c72c..fa2ff04acc 100644 --- a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageFileListPanelComponent.swift +++ b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageFileListPanelComponent.swift @@ -328,7 +328,7 @@ private final class FileListItemComponent: Component { checkLayer.setSelected(isSelected, animated: false) checkLayer.setNeedsDisplay() } - transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize))) + transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: component.sideInset + 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize))) } else { if let checkLayer = self.checkLayer { self.checkLayer = nil diff --git a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StoragePeerListPanelComponent.swift b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StoragePeerListPanelComponent.swift index 419e828be8..5ce3197d04 100644 --- a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StoragePeerListPanelComponent.swift +++ b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StoragePeerListPanelComponent.swift @@ -195,7 +195,7 @@ private final class PeerListItemComponent: Component { checkLayer.setSelected(isSelected, animated: false) checkLayer.setNeedsDisplay() } - transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize))) + transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: component.sideInset + 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize))) } else { if let checkLayer = self.checkLayer { self.checkLayer = nil diff --git a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreen.swift b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreen.swift index 93d1911323..22e158ea09 100644 --- a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreen.swift +++ b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreen.swift @@ -541,7 +541,7 @@ final class StorageUsageScreenComponent: Component { if navigationEditButtonView.superview == nil { self.addSubview(navigationEditButtonView) } - transition.setFrame(view: navigationEditButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - navigationEditButtonSize.width, y: environment.statusBarHeight), size: navigationEditButtonSize)) + transition.setFrame(view: navigationEditButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - environment.safeInsets.right - navigationEditButtonSize.width, y: environment.statusBarHeight), size: navigationEditButtonSize)) } let navigationDoneButtonSize = self.navigationDoneButton.update( @@ -563,7 +563,7 @@ final class StorageUsageScreenComponent: Component { if navigationDoneButtonView.superview == nil { self.addSubview(navigationDoneButtonView) } - transition.setFrame(view: navigationDoneButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - navigationDoneButtonSize.width, y: environment.statusBarHeight), size: navigationDoneButtonSize)) + transition.setFrame(view: navigationDoneButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - environment.safeInsets.right - navigationDoneButtonSize.width, y: environment.statusBarHeight), size: navigationDoneButtonSize)) } let navigationRightButtonMaxWidth: CGFloat = max(navigationEditButtonSize.width, navigationDoneButtonSize.width) @@ -791,6 +791,7 @@ final class StorageUsageScreenComponent: Component { if !self.isOtherCategoryExpanded { var otherSum: CGFloat = 0.0 + var otherRealSum: CGFloat = 0.0 for i in 0 ..< chartItems.count { if otherCategories.contains(chartItems[i].id) { var itemValue = chartItems[i].value @@ -798,6 +799,7 @@ final class StorageUsageScreenComponent: Component { itemValue = max(itemValue, 0.01) } otherSum += itemValue + otherRealSum += chartItems[i].displayValue if case .misc = chartItems[i].id { } else { chartItems[i].value = 0.0 @@ -806,6 +808,7 @@ final class StorageUsageScreenComponent: Component { } if let index = chartItems.firstIndex(where: { $0.id == .misc }) { chartItems[index].value = otherSum + chartItems[index].displayValue = otherRealSum } } @@ -2106,7 +2109,7 @@ final class StorageUsageScreenComponent: Component { } }))) } else { - subItems.append(.custom(MultiplePeerAvatarsContextItem(context: context, peers: peerExceptions.prefix(3).map { EnginePeer($0.peer.peer) }, action: { c, _ in + subItems.append(.custom(MultiplePeerAvatarsContextItem(context: context, peers: peerExceptions.prefix(3).map { EnginePeer($0.peer.peer) }, totalCount: peerExceptions.count, action: { c, _ in c.dismiss(completion: { }) @@ -2185,11 +2188,13 @@ private final class StorageUsageContextReferenceContentSource: ContextReferenceC final class MultiplePeerAvatarsContextItem: ContextMenuCustomItem { fileprivate let context: AccountContext fileprivate let peers: [EnginePeer] + fileprivate let totalCount: Int fileprivate let action: (ContextControllerProtocol, @escaping (ContextMenuActionResult) -> Void) -> Void - init(context: AccountContext, peers: [EnginePeer], action: @escaping (ContextControllerProtocol, @escaping (ContextMenuActionResult) -> Void) -> Void) { + init(context: AccountContext, peers: [EnginePeer], totalCount: Int, action: @escaping (ContextControllerProtocol, @escaping (ContextMenuActionResult) -> Void) -> Void) { self.context = context self.peers = peers + self.totalCount = totalCount self.action = action } @@ -2296,7 +2301,7 @@ private final class MultiplePeerAvatarsContextItemNode: ASDisplayNode, ContextMe let calculatedWidth = min(constrainedWidth, 250.0) let textFont = Font.regular(self.presentationData.listsFontSize.baseDisplaySize) - let text: String = self.presentationData.strings.CacheEvictionMenu_CategoryExceptions(Int32(self.item.peers.count)) + let text: String = self.presentationData.strings.CacheEvictionMenu_CategoryExceptions(Int32(self.item.totalCount)) self.textNode.attributedText = NSAttributedString(string: text, font: textFont, textColor: self.presentationData.theme.contextMenu.primaryColor) let textSize = self.textNode.updateLayout(CGSize(width: calculatedWidth - sideInset - rightTextInset, height: .greatestFiniteMagnitude)) diff --git a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreenSelectionPanelComponent.swift b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreenSelectionPanelComponent.swift index edcaa66582..09fd43a2fc 100644 --- a/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreenSelectionPanelComponent.swift +++ b/submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreenSelectionPanelComponent.swift @@ -101,7 +101,7 @@ final class StorageUsageScreenSelectionPanelComponent: Component { } - let backgroundFrame = CGRect(origin: CGPoint(x: 0.0, y: 0.0), size: CGSize(width: availableSize.height, height: height)) + let backgroundFrame = CGRect(origin: CGPoint(x: 0.0, y: 0.0), size: CGSize(width: availableSize.width, height: height)) transition.setFrame(view: self.backgroundView, frame: backgroundFrame) self.backgroundView.update(size: backgroundFrame.size, transition: transition.containedViewLayoutTransition) diff --git a/submodules/Utils/DarwinDirStat/BUILD b/submodules/Utils/DarwinDirStat/BUILD new file mode 100644 index 0000000000..1505fd502e --- /dev/null +++ b/submodules/Utils/DarwinDirStat/BUILD @@ -0,0 +1,21 @@ + +objc_library( + name = "DarwinDirStat", + enable_modules = True, + module_name = "DarwinDirStat", + srcs = glob([ + "Sources/*.m", + ]), + hdrs = glob([ + "PublicHeaders/**/*.h", + ]), + includes = [ + "PublicHeaders", + ], + sdk_frameworks = [ + "Foundation", + ], + visibility = [ + "//visibility:public", + ], +) diff --git a/submodules/Utils/DarwinDirStat/Package.swift b/submodules/Utils/DarwinDirStat/Package.swift new file mode 100644 index 0000000000..4011c1c65b --- /dev/null +++ b/submodules/Utils/DarwinDirStat/Package.swift @@ -0,0 +1,32 @@ +// swift-tools-version:5.5 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "DarwinDirStat", + platforms: [.macOS(.v10_12)], + products: [ + // Products define the executables and libraries a package produces, and make them visible to other packages. + .library( + name: "DarwinDirStat", + targets: ["DarwinDirStat"]), + ], + dependencies: [ + // Dependencies declare other packages that this package depends on. + // .package(url: /* package url */, from: "1.0.0"), + ], + targets: [ + // Targets are the basic building blocks of a package. A target can define a module or a test suite. + // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "DarwinDirStat", + dependencies: [], + path: ".", + exclude: ["BUILD"], + publicHeadersPath: "PublicHeaders", + cSettings: [ + .headerSearchPath("PublicHeaders") + ]), + ] +) diff --git a/submodules/Utils/DarwinDirStat/PublicHeaders/DarwinDirStat/DarwinDirStat.h b/submodules/Utils/DarwinDirStat/PublicHeaders/DarwinDirStat/DarwinDirStat.h new file mode 100644 index 0000000000..6df75730b7 --- /dev/null +++ b/submodules/Utils/DarwinDirStat/PublicHeaders/DarwinDirStat/DarwinDirStat.h @@ -0,0 +1,14 @@ +#ifndef DarwinDirStat_h +#define DarwinDirStat_h + +#import + +struct darwin_dirstat { + off_t total_size; + uint64_t descendants; +}; + +int dirstat_np(const char *path, int flags, struct darwin_dirstat *ds, size_t dirstat_size); + + +#endif diff --git a/submodules/Utils/DarwinDirStat/Sources/DarwinDirStat.m b/submodules/Utils/DarwinDirStat/Sources/DarwinDirStat.m new file mode 100644 index 0000000000..1c7866bdaf --- /dev/null +++ b/submodules/Utils/DarwinDirStat/Sources/DarwinDirStat.m @@ -0,0 +1,2 @@ +#import +