diff --git a/submodules/Camera/Sources/Camera.swift b/submodules/Camera/Sources/Camera.swift index 9a1ec35ec6..249d6d4e7a 100644 --- a/submodules/Camera/Sources/Camera.swift +++ b/submodules/Camera/Sources/Camera.swift @@ -174,6 +174,10 @@ private final class CameraContext { object: self.session.session ) } + + deinit { + Logger.shared.log("CameraContext", "deinit") + } private var isSessionRunning = false func startCapture() { diff --git a/submodules/Postbox/Sources/SqliteValueBox.swift b/submodules/Postbox/Sources/SqliteValueBox.swift index 56d63d63eb..7b63cd186d 100644 --- a/submodules/Postbox/Sources/SqliteValueBox.swift +++ b/submodules/Postbox/Sources/SqliteValueBox.swift @@ -199,9 +199,12 @@ public final class SqliteValueBox: ValueBox { private var fullTextMatchGlobalStatements: [Int32 : SqlitePreparedStatement] = [:] private var fullTextMatchCollectionStatements: [Int32 : SqlitePreparedStatement] = [:] private var fullTextMatchCollectionTagsStatements: [Int32 : SqlitePreparedStatement] = [:] + private var preparedPageCountStatement: SqlitePreparedStatement? private var secureDeleteEnabled: Bool = false + private var pageSize: Int? + private let checkpoints = MetaDisposable() private let queue: Queue @@ -452,6 +455,8 @@ public final class SqliteValueBox: ValueBox { assert(resultCode) resultCode = database.execute("PRAGMA cipher_memory_security = OFF") assert(resultCode) + + self.pageSize = Int(self.runPragma(database, "page_size")) postboxLog("Did set up pragmas") @@ -1539,6 +1544,32 @@ public final class SqliteValueBox: ValueBox { return resultStatement } + public func getDatabaseSize() -> Int { + precondition(self.queue.isCurrent()) + + guard let pageSize = self.pageSize else { + return 0 + } + + let preparedStatement: SqlitePreparedStatement + if let current = self.preparedPageCountStatement { + preparedStatement = current + } else { + var statement: OpaquePointer? = nil + let status = sqlite3_prepare_v2(database.handle, "PRAGMA page_count", -1, &statement, nil) + precondition(status == SQLITE_OK) + preparedStatement = SqlitePreparedStatement(statement: statement) + self.preparedPageCountStatement = preparedStatement + } + + preparedStatement.reset() + + let _ = preparedStatement.step(handle: database.handle, pathToRemoveOnError: self.removeDatabaseOnError ? self.databasePath : nil) + let value = preparedStatement.int64At(0) + + return Int(value) * pageSize + } + public func get(_ table: ValueBoxTable, key: ValueBoxKey) -> ReadBuffer? { precondition(self.queue.isCurrent()) if let _ = self.tables[table.id] { @@ -2245,6 +2276,11 @@ public final class SqliteValueBox: ValueBox { statement.destroy() } self.fullTextMatchCollectionTagsStatements.removeAll() + + if let preparedPageCountStatement = self.preparedPageCountStatement { + self.preparedPageCountStatement = nil + preparedPageCountStatement.destroy() + } } public func removeAllFromTable(_ table: ValueBoxTable) { diff --git a/submodules/Postbox/Sources/TimeBasedCleanup.swift b/submodules/Postbox/Sources/TimeBasedCleanup.swift index e1a9c01485..cf848ef16f 100644 --- a/submodules/Postbox/Sources/TimeBasedCleanup.swift +++ b/submodules/Postbox/Sources/TimeBasedCleanup.swift @@ -43,7 +43,7 @@ public func printOpenFiles() { private final class TempScanDatabase { private let queue: Queue - private let valueBox: SqliteValueBox + let valueBox: SqliteValueBox private let accessTimeTable: ValueBoxTable @@ -129,7 +129,7 @@ private final class TempScanDatabase { } } -private func scanFiles(at path: String, olderThan minTimestamp: Int32, includeSubdirectories: Bool, performSizeMapping: Bool, tempDatabase: TempScanDatabase) -> ScanFilesResult { +private func scanFiles(at path: String, olderThan minTimestamp: Int32, includeSubdirectories: Bool, performSizeMapping: Bool, tempDatabase: TempScanDatabase, reportMemoryUsageInterval: Int, reportMemoryUsageRemaining: inout Int) -> ScanFilesResult { var result = ScanFilesResult() var subdirectories: [String] = [] @@ -171,6 +171,13 @@ private func scanFiles(at path: String, olderThan minTimestamp: Int32, includeSu result.totalSize += UInt64(value.st_size) if performSizeMapping { tempDatabase.add(pathBuffer: pathBuffer, pathSize: strnlen(pathBuffer, 1024), size: Int64(value.st_size), timestamp: Int32(value.st_mtimespec.tv_sec)) + + reportMemoryUsageRemaining -= 1 + if reportMemoryUsageRemaining <= 0 { + reportMemoryUsageRemaining = reportMemoryUsageInterval + + postboxLog("TimeBasedCleanup in-memory size: \(tempDatabase.valueBox.getDatabaseSize() / (1024 * 1024)) MB") + } } } } @@ -181,7 +188,7 @@ private func scanFiles(at path: String, olderThan minTimestamp: Int32, includeSu if includeSubdirectories { for subPath in subdirectories { - let subResult = scanFiles(at: subPath, olderThan: minTimestamp, includeSubdirectories: true, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase) + let subResult = scanFiles(at: subPath, olderThan: minTimestamp, includeSubdirectories: true, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase, reportMemoryUsageInterval: reportMemoryUsageInterval, reportMemoryUsageRemaining: &reportMemoryUsageRemaining) result.totalSize += subResult.totalSize result.unlinkedCount += subResult.unlinkedCount } @@ -352,6 +359,10 @@ private final class TimeBasedCleanupImpl { let queue = Queue(name: "TimeBasedCleanupScan", qos: .background) queue.async { let tempDirectory = TempBox.shared.tempDirectory() + let randomId = UInt32.random(in: 0 ... UInt32.max) + + postboxLog("TimeBasedCleanup: reset scan id: \(randomId)") + guard let tempDatabase = TempScanDatabase(queue: queue, basePath: tempDirectory.path) else { postboxLog("TimeBasedCleanup: couldn't create temp database at \(tempDirectory.path)") subscriber.putCompletion() @@ -363,6 +374,9 @@ private final class TimeBasedCleanupImpl { var removedGeneralCount: Int = 0 let removedGeneralLimitCount: Int = 0 + let reportMemoryUsageInterval = 100 + var reportMemoryUsageRemaining: Int = reportMemoryUsageInterval + let startTime = CFAbsoluteTimeGetCurrent() var paths: [String] = [] @@ -384,7 +398,6 @@ private final class TimeBasedCleanupImpl { totalApproximateSize += statForDirectory(path: path) } - if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: totalSizeBasedPath), includingPropertiesForKeys: [.fileSizeKey, .fileResourceIdentifierKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) { var fileIds = Set() loop: for url in enumerator { @@ -409,11 +422,18 @@ private final class TimeBasedCleanupImpl { if totalApproximateSize <= bytesLimit { performSizeMapping = false } + #if DEBUG + if "".isEmpty { + performSizeMapping = true + } + #endif + + print("TimeBasedCleanup: id: \(randomId) performSizeMapping: \(performSizeMapping)") let oldestShortLivedTimestamp = timestamp - shortLived let oldestGeneralTimestamp = timestamp - general for path in shortLivedPaths { - let scanResult = scanFiles(at: path, olderThan: oldestShortLivedTimestamp, includeSubdirectories: true, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase) + let scanResult = scanFiles(at: path, olderThan: oldestShortLivedTimestamp, includeSubdirectories: true, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase, reportMemoryUsageInterval: reportMemoryUsageInterval, reportMemoryUsageRemaining: &reportMemoryUsageRemaining) if !paths.contains(path) { paths.append(path) } @@ -424,7 +444,7 @@ private final class TimeBasedCleanupImpl { if general < Int32.max { for path in generalPaths { - let scanResult = scanFiles(at: path, olderThan: oldestGeneralTimestamp, includeSubdirectories: true, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase) + let scanResult = scanFiles(at: path, olderThan: oldestGeneralTimestamp, includeSubdirectories: true, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase, reportMemoryUsageInterval: reportMemoryUsageInterval, reportMemoryUsageRemaining: &reportMemoryUsageRemaining) if !paths.contains(path) { paths.append(path) } @@ -434,7 +454,7 @@ private final class TimeBasedCleanupImpl { } if gigabytesLimit < Int32.max { - let scanResult = scanFiles(at: totalSizeBasedPath, olderThan: 0, includeSubdirectories: false, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase) + let scanResult = scanFiles(at: totalSizeBasedPath, olderThan: 0, includeSubdirectories: false, performSizeMapping: performSizeMapping, tempDatabase: tempDatabase, reportMemoryUsageInterval: reportMemoryUsageInterval, reportMemoryUsageRemaining: &reportMemoryUsageRemaining) if !paths.contains(totalSizeBasedPath) { paths.append(totalSizeBasedPath) }