Storage management improvements

submodules/Components/SolidRoundedButtonComponent/Sources/SolidRoundedButtonComponent.swift

@@ -148,6 +148,7 @@ public final class SolidRoundedButtonComponent: Component {
                 button.gloss = component.gloss
                 
                 button.isEnabled = component.isEnabled
+                button.isUserInteractionEnabled = component.isEnabled
                 
                 button.updateTheme(component.theme)
                 let height = button.updateLayout(width: availableSize.width, transition: .immediate)

submodules/MtProtoKit/PublicHeaders/MtProtoKit/MTProto.h

@@ -77,11 +77,3 @@
 + (NSData *)_manuallyEncryptedMessage:(NSData *)preparedData messageId:(int64_t)messageId authKey:(MTDatacenterAuthKey *)authKey;
 
 @end
-
-//#define DIRSTAT_FAST_ONLY 0x1
-struct darwin_dirstat {
-    off_t total_size;
-    uint64_t descendants;
-};
-
-int dirstat_np(const char *path, int flags, struct darwin_dirstat *ds, size_t dirstat_size);

submodules/Postbox/Sources/MediaBox.swift

@@ -211,7 +211,7 @@ public final class MediaBox {
         self.timeBasedCleanup.setMaxStoreTimes(general: general, shortLived: shortLived, gigabytesLimit: gigabytesLimit)
     }
     
-    private static func idForFileName(name: String) -> String {
+    public static func idForFileName(name: String) -> String {
         if name.hasSuffix("_partial.meta") {
             return String(name[name.startIndex ..< name.index(name.endIndex, offsetBy: -13)])
         } else if name.hasSuffix("_partial") {
@@ -1291,6 +1291,39 @@ public final class MediaBox {
             
             let scanContext = ScanFilesContext(path: basePath)
             
+            func processStale(nextId: Data?) {
+                let _ = (storageBox.enumerateItems(startingWith: nextId, limit: 1000)
+                |> deliverOn(processQueue)).start(next: { ids, realNextId in
+                    var staleIds: [Data] = []
+                    
+                    for id in ids {
+                        if let name = String(data: id, encoding: .utf8) {
+                            if self.resourceUsage(id: MediaResourceId(name)) == 0 {
+                                staleIds.append(id)
+                            }
+                        } else {
+                            staleIds.append(id)
+                        }
+                    }
+                    
+                    if !staleIds.isEmpty {
+                        storageBox.remove(ids: staleIds)
+                    }
+                    
+                    if realNextId == nil {
+                        completion()
+                    } else {
+                        if lowImpact {
+                            processQueue.after(0.4, {
+                                processStale(nextId: realNextId)
+                            })
+                        } else {
+                            processStale(nextId: realNextId)
+                        }
+                    }
+                })
+            }
+            
             func processNext() {
                 processQueue.async {
                     if isCancelled {
@@ -1299,7 +1332,7 @@ public final class MediaBox {
                     
                     let results = scanContext.nextBatch(count: 32000)
                     if results.isEmpty {
-                        completion()
+                        processStale(nextId: nil)
                         return
                     }
                     

submodules/Postbox/Sources/StorageBox/StorageBox.swift

@@ -577,6 +577,44 @@ public final class StorageBox {
             return result
         }
         
+        func enumerateItems(startingWith startId: Data?, limit: Int) -> (ids: [Data], nextStartId: Data?) {
+            self.valueBox.begin()
+            
+            let startKey: ValueBoxKey
+            if let startId = startId, startId.count == 16 {
+                startKey = ValueBoxKey(length: 16)
+                startKey.setData(0, value: startId)
+            } else {
+                startKey = ValueBoxKey(length: 1)
+                startKey.setUInt8(0, value: 0)
+            }
+            
+            let endKey = ValueBoxKey(length: 16)
+            for i in 0 ..< 16 {
+                endKey.setUInt8(i, value: 0xff)
+            }
+            
+            var ids: [Data] = []
+            var nextKey: ValueBoxKey?
+            self.valueBox.range(self.hashIdToInfoTable, start: startKey, end: endKey, values: { key, value in
+                nextKey = key
+                
+                let info = ItemInfo(buffer: value)
+                ids.append(info.id)
+                
+                return true
+            }, limit: limit)
+            
+            self.valueBox.commit()
+            
+            var nextId = nextKey?.getData(0, length: 16)
+            if nextId == startId {
+                nextId = nil
+            }
+            
+            return (ids, nextId)
+        }
+        
         func all() -> [Entry] {
             var result: [Entry] = []
             
@@ -912,4 +950,13 @@ public final class StorageBox {
             impl.reset()
         }
     }
+    
+    public func enumerateItems(startingWith startId: Data?, limit: Int) -> Signal<(ids: [Data], nextStartId: Data?), NoError> {
+        return self.impl.signalWith { impl, subscriber in
+            subscriber.putNext(impl.enumerateItems(startingWith: startId, limit: limit))
+            subscriber.putCompletion()
+            
+            return EmptyDisposable
+        }
+    }
 }

submodules/Postbox/Sources/TimeBasedCleanup.swift

@@ -39,32 +39,6 @@ public func printOpenFiles() {
     }
 }
 
-/*
- +(void) lsof
- {
-     int flags;
-     int fd;
-     char buf[MAXPATHLEN+1] ;
-     int n = 1 ;
-
-     for (fd = 0; fd < (int) FD_SETSIZE; fd++) {
-         errno = 0;
-         flags = fcntl(fd, F_GETFD, 0);
-         if (flags == -1 && errno) {
-             if (errno != EBADF) {
-                 return ;
-             }
-             else
-                 continue;
-         }
-         fcntl(fd , F_GETPATH, buf ) ;
-         NSLog( @"File Descriptor %d number %d in use for: %s",fd,n , buf ) ;
-         ++n ;
-     }
- }
- 
- */
-
 private func scanFiles(at path: String, olderThan minTimestamp: Int32, inodes: inout [InodeInfo]) -> ScanFilesResult {
     var result = ScanFilesResult()
     
@@ -113,7 +87,7 @@ private func scanFiles(at path: String, olderThan minTimestamp: Int32, inodes: i
     return result
 }
 
-private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UInt64) {
+private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UInt64, mainStoragePath: String, storageBox: StorageBox) {
     var removedSize: UInt64 = 0
     
     inodes.sort(by: { lhs, rhs in
@@ -139,7 +113,10 @@ private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UI
         free(pathBuffer)
     }
     
+    var unlinkedResourceIds: [Data] = []
+    
     for path in paths {
+        let isMainPath = path == mainStoragePath
         if let dp = opendir(path) {
             while true {
                 guard let dirp = readdir(dp) else {
@@ -162,6 +139,17 @@ private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UI
                 var value = stat()
                 if stat(pathBuffer, &value) == 0 {
                     if inodesToDelete.contains(value.st_ino) {
+                        if isMainPath {
+                            let nameLength = strnlen(&dirp.pointee.d_name.0, 1024)
+                            let nameData = Data(bytesNoCopy: &dirp.pointee.d_name.0, count: Int(nameLength), deallocator: .none)
+                            withExtendedLifetime(nameData, {
+                                if let fileName = String(data: nameData, encoding: .utf8) {
+                                    if let idData = MediaBox.idForFileName(name: fileName).data(using: .utf8) {
+                                        unlinkedResourceIds.append(idData)
+                                    }
+                                }
+                            })
+                        }
                         unlink(pathBuffer)
                     }
                 }
@@ -169,6 +157,10 @@ private func mapFiles(paths: [String], inodes: inout [InodeInfo], removeSize: UI
             closedir(dp)
         }
     }
+    
+    if !unlinkedResourceIds.isEmpty {
+        storageBox.remove(ids: unlinkedResourceIds)
+    }
 }
 
 private final class TimeBasedCleanupImpl {
@@ -226,6 +218,7 @@ private final class TimeBasedCleanupImpl {
         let generalPaths = self.generalPaths
         let totalSizeBasedPath = self.totalSizeBasedPath
         let shortLivedPaths = self.shortLivedPaths
+        let storageBox = self.storageBox
         let scanOnce = Signal<Never, NoError> { subscriber in
             DispatchQueue.global(qos: .background).async {
                 var removedShortLivedCount: Int = 0
@@ -238,7 +231,12 @@ private final class TimeBasedCleanupImpl {
                 var paths: [String] = []
                 
                 let timestamp = Int32(Date().timeIntervalSince1970)
+                
+                /*#if DEBUG
+                let bytesLimit: UInt64 = 10 * 1024 * 1024
+                #else*/
                 let bytesLimit = UInt64(gigabytesLimit) * 1024 * 1024 * 1024
+                //#endif
                 
                 let oldestShortLivedTimestamp = timestamp - shortLived
                 let oldestGeneralTimestamp = timestamp - general
@@ -270,13 +268,9 @@ private final class TimeBasedCleanupImpl {
                 }
                 
                 if totalLimitSize > bytesLimit {
-                    mapFiles(paths: paths, inodes: &inodes, removeSize: totalLimitSize - bytesLimit)
+                    mapFiles(paths: paths, inodes: &inodes, removeSize: totalLimitSize - bytesLimit, mainStoragePath: totalSizeBasedPath, storageBox: storageBox)
                 }
                 
-                #if DEBUG
-                //printOpenFiles()
-                #endif
-                
                 if removedShortLivedCount != 0 || removedGeneralCount != 0 || removedGeneralLimitCount != 0 {
                     postboxLog("[TimeBasedCleanup] \(CFAbsoluteTimeGetCurrent() - startTime) s removed \(removedShortLivedCount) short-lived files, \(removedGeneralCount) general files, \(removedGeneralLimitCount) limit files")
                 }

submodules/TelegramCore/BUILD

@@ -47,6 +47,7 @@ swift_library(
         "//submodules/Reachability:Reachability",
         "//submodules/ManagedFile:ManagedFile",
         "//submodules/Utils/RangeSet:RangeSet",
+        "//submodules/Utils/DarwinDirStat",
     ],
     visibility = [
         "//visibility:public",

submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift

@@ -2,6 +2,7 @@ import Foundation
 import Postbox
 import SwiftSignalKit
 import MtProtoKit
+import DarwinDirStat
 
 public enum PeerCacheUsageCategory: Int32 {
     case image = 0

submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageFileListPanelComponent.swift

@@ -328,7 +328,7 @@ private final class FileListItemComponent: Component {
                     checkLayer.setSelected(isSelected, animated: false)
                     checkLayer.setNeedsDisplay()
                 }
-                transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize)))
+                transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: component.sideInset + 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize)))
             } else {
                 if let checkLayer = self.checkLayer {
                     self.checkLayer = nil

submodules/TelegramUI/Components/StorageUsageScreen/Sources/StoragePeerListPanelComponent.swift

@@ -195,7 +195,7 @@ private final class PeerListItemComponent: Component {
                     checkLayer.setSelected(isSelected, animated: false)
                     checkLayer.setNeedsDisplay()
                 }
-                transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize)))
+                transition.setFrame(layer: checkLayer, frame: CGRect(origin: CGPoint(x: component.sideInset + 20.0, y: floor((height - checkSize) / 2.0)), size: CGSize(width: checkSize, height: checkSize)))
             } else {
                 if let checkLayer = self.checkLayer {
                     self.checkLayer = nil

submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreen.swift

@@ -541,7 +541,7 @@ final class StorageUsageScreenComponent: Component {
                 if navigationEditButtonView.superview == nil {
                     self.addSubview(navigationEditButtonView)
                 }
-                transition.setFrame(view: navigationEditButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - navigationEditButtonSize.width, y: environment.statusBarHeight), size: navigationEditButtonSize))
+                transition.setFrame(view: navigationEditButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - environment.safeInsets.right - navigationEditButtonSize.width, y: environment.statusBarHeight), size: navigationEditButtonSize))
             }
             
             let navigationDoneButtonSize = self.navigationDoneButton.update(
@@ -563,7 +563,7 @@ final class StorageUsageScreenComponent: Component {
                 if navigationDoneButtonView.superview == nil {
                     self.addSubview(navigationDoneButtonView)
                 }
-                transition.setFrame(view: navigationDoneButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - navigationDoneButtonSize.width, y: environment.statusBarHeight), size: navigationDoneButtonSize))
+                transition.setFrame(view: navigationDoneButtonView, frame: CGRect(origin: CGPoint(x: availableSize.width - 12.0 - environment.safeInsets.right - navigationDoneButtonSize.width, y: environment.statusBarHeight), size: navigationDoneButtonSize))
             }
             
             let navigationRightButtonMaxWidth: CGFloat = max(navigationEditButtonSize.width, navigationDoneButtonSize.width)
@@ -791,6 +791,7 @@ final class StorageUsageScreenComponent: Component {
             
             if !self.isOtherCategoryExpanded {
                 var otherSum: CGFloat = 0.0
+                var otherRealSum: CGFloat = 0.0
                 for i in 0 ..< chartItems.count {
                     if otherCategories.contains(chartItems[i].id) {
                         var itemValue = chartItems[i].value
@@ -798,6 +799,7 @@ final class StorageUsageScreenComponent: Component {
                             itemValue = max(itemValue, 0.01)
                         }
                         otherSum += itemValue
+                        otherRealSum += chartItems[i].displayValue
                         if case .misc = chartItems[i].id {
                         } else {
                             chartItems[i].value = 0.0
@@ -806,6 +808,7 @@ final class StorageUsageScreenComponent: Component {
                 }
                 if let index = chartItems.firstIndex(where: { $0.id == .misc }) {
                     chartItems[index].value = otherSum
+                    chartItems[index].displayValue = otherRealSum
                 }
             }
             
@@ -2106,7 +2109,7 @@ final class StorageUsageScreenComponent: Component {
                         }
                     })))
                 } else {
-                    subItems.append(.custom(MultiplePeerAvatarsContextItem(context: context, peers: peerExceptions.prefix(3).map { EnginePeer($0.peer.peer) }, action: { c, _ in
+                    subItems.append(.custom(MultiplePeerAvatarsContextItem(context: context, peers: peerExceptions.prefix(3).map { EnginePeer($0.peer.peer) }, totalCount: peerExceptions.count, action: { c, _ in
                         c.dismiss(completion: {
                             
                         })
@@ -2185,11 +2188,13 @@ private final class StorageUsageContextReferenceContentSource: ContextReferenceC
 final class MultiplePeerAvatarsContextItem: ContextMenuCustomItem {
     fileprivate let context: AccountContext
     fileprivate let peers: [EnginePeer]
+    fileprivate let totalCount: Int
     fileprivate let action: (ContextControllerProtocol, @escaping (ContextMenuActionResult) -> Void) -> Void
 
-    init(context: AccountContext, peers: [EnginePeer], action: @escaping (ContextControllerProtocol, @escaping (ContextMenuActionResult) -> Void) -> Void) {
+    init(context: AccountContext, peers: [EnginePeer], totalCount: Int, action: @escaping (ContextControllerProtocol, @escaping (ContextMenuActionResult) -> Void) -> Void) {
         self.context = context
         self.peers = peers
+        self.totalCount = totalCount
         self.action = action
     }
 
@@ -2296,7 +2301,7 @@ private final class MultiplePeerAvatarsContextItemNode: ASDisplayNode, ContextMe
         let calculatedWidth = min(constrainedWidth, 250.0)
 
         let textFont = Font.regular(self.presentationData.listsFontSize.baseDisplaySize)
-        let text: String = self.presentationData.strings.CacheEvictionMenu_CategoryExceptions(Int32(self.item.peers.count))
+        let text: String = self.presentationData.strings.CacheEvictionMenu_CategoryExceptions(Int32(self.item.totalCount))
         self.textNode.attributedText = NSAttributedString(string: text, font: textFont, textColor: self.presentationData.theme.contextMenu.primaryColor)
 
         let textSize = self.textNode.updateLayout(CGSize(width: calculatedWidth - sideInset - rightTextInset, height: .greatestFiniteMagnitude))

submodules/TelegramUI/Components/StorageUsageScreen/Sources/StorageUsageScreenSelectionPanelComponent.swift

@@ -101,7 +101,7 @@ final class StorageUsageScreenSelectionPanelComponent: Component {
             }
             
             
-            let backgroundFrame = CGRect(origin: CGPoint(x: 0.0, y: 0.0), size: CGSize(width: availableSize.height, height: height))
+            let backgroundFrame = CGRect(origin: CGPoint(x: 0.0, y: 0.0), size: CGSize(width: availableSize.width, height: height))
             transition.setFrame(view: self.backgroundView, frame: backgroundFrame)
             self.backgroundView.update(size: backgroundFrame.size, transition: transition.containedViewLayoutTransition)
             

submodules/Utils/DarwinDirStat/BUILD

@@ -0,0 +1,21 @@
+
+objc_library(
+    name = "DarwinDirStat",
+    enable_modules = True,
+    module_name = "DarwinDirStat",
+    srcs = glob([
+        "Sources/*.m",
+    ]),
+    hdrs = glob([
+        "PublicHeaders/**/*.h",
+    ]),
+    includes = [
+        "PublicHeaders",
+    ],
+    sdk_frameworks = [
+      "Foundation",
+    ],
+    visibility = [
+        "//visibility:public",
+    ],
+)

submodules/Utils/DarwinDirStat/Package.swift

@@ -0,0 +1,32 @@
+// swift-tools-version:5.5
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "DarwinDirStat",
+    platforms: [.macOS(.v10_12)],
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "DarwinDirStat",
+            targets: ["DarwinDirStat"]),
+    ],
+    dependencies: [
+        // Dependencies declare other packages that this package depends on.
+        // .package(url: /* package url */, from: "1.0.0"),
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+        // Targets can depend on other targets in this package, and on products in packages this package depends on.
+        .target(
+            name: "DarwinDirStat",
+            dependencies: [],
+            path: ".",
+            exclude: ["BUILD"],
+            publicHeadersPath: "PublicHeaders",
+            cSettings: [
+                .headerSearchPath("PublicHeaders")
+            ]),
+    ]
+)

submodules/Utils/DarwinDirStat/PublicHeaders/DarwinDirStat/DarwinDirStat.h

@@ -0,0 +1,14 @@
+#ifndef DarwinDirStat_h
+#define DarwinDirStat_h
+
+#import <Foundation/Foundation.h>
+
+struct darwin_dirstat {
+    off_t total_size;
+    uint64_t descendants;
+};
+
+int dirstat_np(const char *path, int flags, struct darwin_dirstat *ds, size_t dirstat_size);
+
+
+#endif

submodules/Utils/DarwinDirStat/Sources/DarwinDirStat.m

@@ -0,0 +1,2 @@
+#import <DarwinDirStat/DarwinDirStat.h>
+