From b34244cae86d99fc081945970a03d44bea1d65d4 Mon Sep 17 00:00:00 2001 From: Peter Date: Tue, 26 Jan 2016 16:16:40 +0300 Subject: [PATCH] no message --- Postbox.xcodeproj/project.pbxproj | 36 +- .../xcschemes/PostboxTests.xcscheme | 3 +- Postbox/Coding.swift | 4 + Postbox/LmdbValueBox.swift | 56 ++ Postbox/Media.swift | 12 +- Postbox/MediaCleanupTable.swift | 21 + Postbox/Message.swift | 77 ++- Postbox/MessageHistoryHole.swift | 14 + Postbox/MessageHistoryIndexTable.swift | 322 +++++++++ Postbox/MessageHistoryTable.swift | 359 ++++++++++ Postbox/MessageHistoryView.swift | 341 +++++++++ Postbox/MessageMediaTable.swift | 210 ++++++ Postbox/MessageView.swift | 428 ------------ Postbox/Peer.swift | 10 +- Postbox/Postbox.swift | 646 ++---------------- Postbox/PostboxTables.swift | 54 +- Postbox/SqliteValueBox.swift | 61 +- Postbox/ValueBox.swift | 3 + PostboxTests/CodingTests.swift | 43 +- .../MessageHistoryIndexTableTests.swift | 427 ++++++++++++ PostboxTests/MessageHistoryTableTests.swift | 306 +++++++++ 21 files changed, 2335 insertions(+), 1098 deletions(-) create mode 100644 Postbox/MediaCleanupTable.swift create mode 100644 Postbox/MessageHistoryHole.swift create mode 100644 Postbox/MessageHistoryIndexTable.swift create mode 100644 Postbox/MessageHistoryTable.swift create mode 100644 Postbox/MessageHistoryView.swift create mode 100644 Postbox/MessageMediaTable.swift delete mode 100644 Postbox/MessageView.swift create mode 100644 PostboxTests/MessageHistoryIndexTableTests.swift create mode 100644 PostboxTests/MessageHistoryTableTests.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 9c8cd5f2b6..dec81ee69a 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -7,7 +7,7 @@ objects = { /* Begin PBXBuildFile section */ - D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageView.swift */; }; + D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; D00E0FB31B84CEDA002E4EB5 /* Display.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D00E0FB21B84CEDA002E4EB5 /* Display.framework */; }; D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; }; @@ -29,11 +29,15 @@ D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; }; D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; }; D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; }; + D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; }; + D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; }; + D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; }; D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; D0977FA21B82930C009994B2 /* PostboxCodingUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */; }; D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA31B829F87009994B2 /* PostboxTables.swift */; }; + D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; }; @@ -45,6 +49,9 @@ D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; }; D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; }; D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; }; + D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; }; + D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; }; + D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -58,7 +65,7 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ - D003E4E51B38DBDB00C22CBC /* MessageView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageView.swift; sourceTree = ""; }; + D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryView.swift; sourceTree = ""; }; D00E0FB21B84CEDA002E4EB5 /* Display.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Display.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/Display.framework"; sourceTree = ""; }; D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; @@ -81,11 +88,15 @@ D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = ""; }; D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = ""; }; D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = ""; }; + D08C71391C501F0700779C0F /* MessageHistoryHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHole.swift; sourceTree = ""; }; + D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = ""; }; + D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = ""; }; D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxCodingUtils.swift; sourceTree = ""; }; D0977FA31B829F87009994B2 /* PostboxTables.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxTables.swift; sourceTree = ""; }; + D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = ""; }; D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; }; @@ -100,6 +111,9 @@ D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = ""; }; D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = ""; }; D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = ""; }; + D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = ""; }; + D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = ""; }; + D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -207,8 +221,9 @@ D0B76BE61B66639F0095CF45 /* DeferredString.swift */, D0E3A7831B28AE0900A402D9 /* Peer.swift */, D0E3A79D1B28B50400A402D9 /* Message.swift */, + D08C71391C501F0700779C0F /* MessageHistoryHole.swift */, D0E3A7A11B28B7DC00A402D9 /* Media.swift */, - D003E4E51B38DBDB00C22CBC /* MessageView.swift */, + D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */, D0D225251B4D84930085E26D /* PeerView.swift */, D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */, D0E3A7811B28ADD000A402D9 /* Postbox.swift */, @@ -218,6 +233,10 @@ D0977F9B1B822DB4009994B2 /* ValueBox.swift */, D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */, D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */, + D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */, + D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */, + D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */, + D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */, D0E3A74D1B28A7E300A402D9 /* Supporting Files */, ); path = Postbox; @@ -239,6 +258,8 @@ children = ( D0E3A75A1B28A7E300A402D9 /* Supporting Files */, D044E15D1B2ACB9C001EE087 /* CodingTests.swift */, + D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */, + D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -367,24 +388,29 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */, D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */, D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */, D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */, + D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */, + D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */, + D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, - D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */, + D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */, D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */, D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */, + D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */, D0D225261B4D84930085E26D /* PeerView.swift in Sources */, D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */, D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */, @@ -401,6 +427,8 @@ buildActionMask = 2147483647; files = ( D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, + D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */, + D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme index 35ff5f38d6..fdd99d4b63 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme @@ -10,7 +10,8 @@ buildConfiguration = "Debug" selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB" - shouldUseLaunchSchemeArgsEnv = "YES"> + shouldUseLaunchSchemeArgsEnv = "YES" + codeCoverageEnabled = "YES"> diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index a90fbc0d63..e084d99829 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -137,6 +137,10 @@ public final class ReadBuffer: MemoryBuffer { self.offset += length } + func skip(length: Int) { + self.offset += length + } + func reset() { self.offset = 0 } diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift index f433362f47..6ad6b4b304 100644 --- a/Postbox/LmdbValueBox.swift +++ b/Postbox/LmdbValueBox.swift @@ -98,6 +98,10 @@ public final class LmdbValueBox: ValueBox { private var sharedTxn: COpaquePointer = nil + private var readQueryTime: CFAbsoluteTime = 0.0 + private var writeQueryTime: CFAbsoluteTime = 0.0 + private var commitTime: CFAbsoluteTime = 0.0 + public init?(basePath: String) { var result = mdb_env_create(&self.env) if result != MDB_SUCCESS { @@ -163,6 +167,16 @@ public final class LmdbValueBox: ValueBox { return LmdbTable(dbi: dbi) } + public func beginStats() { + self.readQueryTime = 0.0 + self.writeQueryTime = 0.0 + self.commitTime = 0.0 + } + + public func endStats() { + print("(LmdbValueBox stats read: \(self.readQueryTime * 1000.0) ms, write: \(self.writeQueryTime * 1000.0) ms, commit: \(self.commitTime * 1000.0) ms") + } + public func begin() { if self.sharedTxn != nil { print("(LmdbValueBox already in transaction)") @@ -176,11 +190,13 @@ public final class LmdbValueBox: ValueBox { } public func commit() { + let startTime = CFAbsoluteTimeGetCurrent() if self.sharedTxn == nil { print("(LmdbValueBox already no current transaction)") } else { let result = mdb_txn_commit(self.sharedTxn) self.sharedTxn = nil + self.commitTime += CFAbsoluteTimeGetCurrent() - startTime if result != MDB_SUCCESS { print("(LmdbValueBox txn_commit failed with \(result))") return @@ -208,6 +224,7 @@ public final class LmdbValueBox: ValueBox { } if let nativeTable = nativeTable { + var startTime = CFAbsoluteTimeGetCurrent() var cursorPtr: COpaquePointer = nil let result = mdb_cursor_open(self.sharedTxn, nativeTable.dbi, &cursorPtr) if result != MDB_SUCCESS { @@ -223,6 +240,10 @@ public final class LmdbValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + var count = 0 if value != nil && value!.0 < end { count++ @@ -230,13 +251,21 @@ public final class LmdbValueBox: ValueBox { } while value != nil && value!.0 < end && count < limit { + startTime = CFAbsoluteTimeGetCurrent() + value = cursor.next() + + currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + if value != nil && value!.0 < end { count++ values(value!.0, value!.1) } } } else { + var startTime = CFAbsoluteTimeGetCurrent() var value = cursor.seekTo(start, forward: false) if value != nil { if value!.0 == start { @@ -244,6 +273,10 @@ public final class LmdbValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + var count = 0 if value != nil && value!.0 > end { count++ @@ -251,7 +284,14 @@ public final class LmdbValueBox: ValueBox { } while value != nil && value!.0 > end && count < limit { + startTime = CFAbsoluteTimeGetCurrent() + value = cursor.previous() + + currentTime = CFAbsoluteTimeGetCurrent() + readQueryTime += currentTime - startTime + startTime = currentTime + if value != nil && value!.0 > end { count++ values(value!.0, value!.1) @@ -264,7 +304,11 @@ public final class LmdbValueBox: ValueBox { } if commit { + let startTime = CFAbsoluteTimeGetCurrent() + self.commit() + + readQueryTime += CFAbsoluteTimeGetCurrent() - startTime } } @@ -275,6 +319,8 @@ public final class LmdbValueBox: ValueBox { } public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + let startTime = CFAbsoluteTimeGetCurrent() + var commit = false if self.sharedTxn == nil { self.begin() @@ -315,6 +361,8 @@ public final class LmdbValueBox: ValueBox { self.commit() } + readQueryTime += CFAbsoluteTimeGetCurrent() - startTime + return resultValue } @@ -323,6 +371,8 @@ public final class LmdbValueBox: ValueBox { } public func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) { + let startTime = CFAbsoluteTimeGetCurrent() + var commit = false if self.sharedTxn == nil { self.begin() @@ -355,9 +405,13 @@ public final class LmdbValueBox: ValueBox { if commit { self.commit() } + + writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } public func remove(table: Int32, key: ValueBoxKey) { + let startTime = CFAbsoluteTimeGetCurrent() + var commit = false if self.sharedTxn == nil { self.begin() @@ -387,6 +441,8 @@ public final class LmdbValueBox: ValueBox { if commit { self.commit() } + + writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } public func drop() { diff --git a/Postbox/Media.swift b/Postbox/Media.swift index a62a74759d..4cd9848945 100644 --- a/Postbox/Media.swift +++ b/Postbox/Media.swift @@ -25,11 +25,13 @@ public struct MediaId: Hashable, CustomStringConvertible { } public init(_ buffer: ReadBuffer) { - self.namespace = 0 - self.id = 0 + var namespace: Int32 = 0 + var id: Int64 = 0 - memcpy(&self.namespace, buffer.memory + buffer.offset, 4) - memcpy(&self.id, buffer.memory + (buffer.offset + 4), 8) + memcpy(&namespace, buffer.memory + buffer.offset, 4) + self.namespace = namespace + memcpy(&id, buffer.memory + (buffer.offset + 4), 8) + self.id = id buffer.offset += 12 } @@ -68,4 +70,6 @@ public func ==(lhs: MediaId, rhs: MediaId) -> Bool { public protocol Media: Coding { var id: MediaId? { get } + + func isEqual(other: Media) -> Bool } diff --git a/Postbox/MediaCleanupTable.swift b/Postbox/MediaCleanupTable.swift new file mode 100644 index 0000000000..d86e3c91a7 --- /dev/null +++ b/Postbox/MediaCleanupTable.swift @@ -0,0 +1,21 @@ +import Foundation + +final class MediaCleanupTable { + let valueBox: ValueBox + let tableId: Int32 + + var debugMedia: [Media] = [] + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + func add(media: Media, sharedEncoder: Encoder = Encoder()) { + debugMedia.append(media) + } + + func debugList() -> [Media] { + return self.debugMedia + } +} \ No newline at end of file diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 62e0829108..54ac816893 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -1,6 +1,6 @@ import Foundation -public struct MessageId: Hashable, CustomStringConvertible { +public struct MessageId: Hashable, Comparable, CustomStringConvertible { public typealias Namespace = Int32 public typealias Id = Int32 @@ -81,6 +81,14 @@ public func ==(lhs: MessageId, rhs: MessageId) -> Bool { return lhs.id == rhs.id && lhs.namespace == rhs.namespace } +public func <(lhs: MessageId, rhs: MessageId) -> Bool { + if lhs.namespace == rhs.namespace { + return lhs.id < rhs.id + } else { + return lhs.namespace < rhs.namespace + } +} + public struct MessageIndex: Equatable, Comparable { public let id: MessageId public let timestamp: Int32 @@ -90,10 +98,23 @@ public struct MessageIndex: Equatable, Comparable { self.timestamp = message.timestamp } + init(_ message: StoreMessage) { + self.id = message.id + self.timestamp = message.timestamp + } + public init(id: MessageId, timestamp: Int32) { self.id = id self.timestamp = timestamp } + + public func predecessor() -> MessageIndex { + return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id - 1), timestamp: self.timestamp) + } + + public func successor() -> MessageIndex { + return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id + 1), timestamp: self.timestamp) + } } public func ==(lhs: MessageIndex, rhs: MessageIndex) -> Bool { @@ -112,12 +133,54 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool { return lhs.id.id < rhs.id.id } -public protocol Message: Coding { - var id: MessageId { get } - var timestamp: Int32 { get } - var text: String { get } - var mediaIds: [MediaId] { get } - var peerIds: [PeerId] { get } +public class Message { + let id: MessageId + let timestamp: Int32 + let text: String + let attributes: [Coding] + let media: [Media] + + init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) { + self.id = id + self.timestamp = timestamp + self.text = text + self.attributes = attributes + self.media = media + } +} + +class StoreMessage { + let id: MessageId + let timestamp: Int32 + let text: String + let attributes: [Coding] + let media: [Media] + + init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) { + self.id = id + self.timestamp = timestamp + self.text = text + self.attributes = attributes + self.media = media + } +} + +class IntermediateMessage { + let id: MessageId + let timestamp: Int32 + let text: String + let attributesData: ReadBuffer + let embeddedMediaData: ReadBuffer + let referencedMedia: [MediaId] + + init(id: MessageId, timestamp: Int32, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) { + self.id = id + self.timestamp = timestamp + self.text = text + self.attributesData = attributesData + self.embeddedMediaData = embeddedMediaData + self.referencedMedia = referencedMedia + } } public struct RenderedMessage: Equatable, Comparable { diff --git a/Postbox/MessageHistoryHole.swift b/Postbox/MessageHistoryHole.swift new file mode 100644 index 0000000000..e43822bba9 --- /dev/null +++ b/Postbox/MessageHistoryHole.swift @@ -0,0 +1,14 @@ +import Foundation + +public struct MessageHistoryHole: Equatable { + let maxIndex: MessageIndex + let min: MessageId.Id + + var id: MessageId { + return maxIndex.id + } +} + +public func ==(lhs: MessageHistoryHole, rhs: MessageHistoryHole) -> Bool { + return lhs.maxIndex == rhs.maxIndex && lhs.min == rhs.min +} diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift new file mode 100644 index 0000000000..feb548012c --- /dev/null +++ b/Postbox/MessageHistoryIndexTable.swift @@ -0,0 +1,322 @@ +import Foundation + +enum HistoryIndexEntry { + case Message(MessageIndex) + case Hole(MessageHistoryHole) +} + +enum HoleFillType { + case UpperToLower + case LowerToUpper + case Complete +} + +private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + var timestamp: Int32 = 0 + value.read(×tamp, offset: 0, length: 4) + let index = MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: key.getInt32(8 + 4)), timestamp: timestamp) + + if type == 0 { + return .Message(index) + } else { + var min: Int32 = 0 + value.read(&min, offset: 0, length: 4) + return .Hole(MessageHistoryHole(maxIndex: index, min: min)) + } +} + +final class MessageHistoryIndexTable { + let valueBox: ValueBox + let tableId: Int32 + + init(valueBox: ValueBox, tableId: Int32) { + self.valueBox = valueBox + self.tableId = tableId + } + + func key(id: MessageId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4 + 4) + key.setInt64(0, value: id.peerId.toInt64()) + key.setInt32(8, value: id.namespace) + key.setInt32(8 + 4, value: id.id) + return key + } + + func lowerBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setInt32(8, value: namespace) + return key + } + + func upperBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setInt32(8, value: namespace) + return key.successor + } + + func addHole(id: MessageId) { + let adjacent = self.adjacentItems(id) + + /* + + 1. [x] nothing + 2. [x] message - * - nothing + 3. [x] nothing - * - message + 4. [x] nothing - * - hole + 5. [x] message - * - message + 6. [x] hole - * - message + 7. [x] message - * - hole + + */ + + if let lowerItem = adjacent.lower, upperItem = adjacent.upper { + switch lowerItem { + case .Hole: + break + case let .Message(lowerMessage): + switch upperItem { + case .Hole: + break + case let .Message(upperMessage): + if lowerMessage.id.id < upperMessage.id.id - 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerMessage.id.id + 1)) + } + break + } + } + } else if let lowerItem = adjacent.lower { + switch lowerItem { + case let .Message(lowerMessage): + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerMessage.id.id + 1)) + case .Hole: + break + } + } else if let upperItem = adjacent.upper { + switch upperItem { + case let .Message(upperMessage): + if upperMessage.id.id > 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: 1)) + } + case .Hole: + break + } + } else { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: 1)) + } + } + + func addMessage(index: MessageIndex) { + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: self.key(index.id).predecessor, end: self.upperBound(index.id.peerId, namespace: index.id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(index.id.peerId, namespace: index.id.namespace, key: key, value: value) + return true + }, limit: 1) + + if let upperItem = upperItem { + switch upperItem { + case let .Hole(upperHole): + self.justRemove(upperHole.id) + if upperHole.maxIndex.id.id > index.id.id + 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: index.id.id + 1)) + } + if upperHole.min <= index.id.id - 1 { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: index.id.peerId, namespace: index.id.namespace, id: index.id.id - 1), timestamp: index.timestamp), min: upperHole.min)) + } + break + case .Message: + break + } + } + + self.justInsertMessage(index) + } + + func removeMessage(id: MessageId) { + let key = self.key(id) + if self.valueBox.exists(self.tableId, key: key) { + self.justRemove(id) + + let adjacent = self.adjacentItems(id) + + if let lowerItem = adjacent.lower, upperItem = adjacent.upper { + switch lowerItem { + case let .Message(lowerMessage): + switch upperItem { + case let .Hole(upperHole): + self.justRemove(upperHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1)) + case .Message: + break + } + case let .Hole(lowerHole): + switch upperItem { + case let .Hole(upperHole): + self.justRemove(lowerHole.id) + self.justRemove(upperHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerHole.min)) + case let .Message(upperMessage): + self.justRemove(lowerHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min)) + } + } + } else if let lowerItem = adjacent.lower { + switch lowerItem { + case let .Hole(lowerHole): + self.justRemove(lowerHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min)) + break + case .Message: + break + } + } else if let upperItem = adjacent.upper { + switch upperItem { + case let .Hole(upperHole): + self.justRemove(upperHole.id) + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: 1)) + break + case .Message: + break + } + } + } + } + + func fillHole(id: MessageId, fillType: HoleFillType, indices: [MessageIndex]) { + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + return true + }, limit: 1) + + let sortedByIdIndices = indices.sort({$0.id < $1.id}) + var remainingIndices = sortedByIdIndices + + if let upperItem = upperItem { + switch upperItem { + case let .Hole(upperHole): + var i = 0 + var minIndexInRange: MessageIndex? + var maxIndexInRange: MessageIndex? + var removedHole = false + while i < remainingIndices.count { + let index = remainingIndices[i] + if index.id.id >= upperHole.min && index.id.id <= upperHole.maxIndex.id.id { + if (fillType == .UpperToLower || fillType == .Complete) && (minIndexInRange == nil || minIndexInRange!.id > index.id) { + minIndexInRange = index + if !removedHole { + removedHole = true + self.justRemove(upperHole.id) + } + } + if (fillType == .LowerToUpper || fillType == .Complete) && (maxIndexInRange == nil || maxIndexInRange!.id < index.id) { + maxIndexInRange = index + if !removedHole { + removedHole = true + self.justRemove(upperHole.id) + } + } + self.justInsertMessage(index) + remainingIndices.removeAtIndex(i) + } else { + i++ + } + } + switch fillType { + case .Complete: + if !removedHole { + self.justRemove(upperHole.id) + } + case .LowerToUpper: + if let maxIndexInRange = maxIndexInRange where maxIndexInRange.id.id != Int32.max && maxIndexInRange.id.id + 1 <= upperHole.maxIndex.id.id { + self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: maxIndexInRange.id.id + 1)) + } + case .UpperToLower: + if let minIndexInRange = minIndexInRange where minIndexInRange.id.id - 1 >= upperHole.min { + self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minIndexInRange.id.id - 1), timestamp: minIndexInRange.timestamp), min: upperHole.min)) + } + } + break + case .Message: + break + } + } + + for index in remainingIndices { + self.addMessage(index) + } + } + + func justInsertHole(hole: MessageHistoryHole) { + let value = WriteBuffer() + var type: Int8 = 1 + var timestamp: Int32 = hole.maxIndex.timestamp + var min: Int32 = hole.min + value.write(&type, offset: 0, length: 1) + value.write(×tamp, offset: 0, length: 4) + value.write(&min, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(hole.id), value: value) + } + + func justInsertMessage(index: MessageIndex) { + let value = WriteBuffer() + var type: Int8 = 0 + var timestamp: Int32 = index.timestamp + value.write(&type, offset: 0, length: 1) + value.write(×tamp, offset: 0, length: 4) + self.valueBox.set(self.tableId, key: self.key(index.id), value: value) + } + + func justRemove(id: MessageId) { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + + func adjacentItems(id: MessageId) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) { + let key = self.key(id) + + var lowerItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: key, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in + lowerItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + return true + }, limit: 1) + + var upperItem: HistoryIndexEntry? + self.valueBox.range(self.tableId, start: key.predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in + upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + return true + }, limit: 1) + + return (lower: lowerItem, upper: upperItem) + } + + func get(id: MessageId) -> HistoryIndexEntry? { + let key = self.key(id) + if let value = self.valueBox.get(self.tableId, key: key) { + return readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value) + } + return nil + } + + func messageExists(id: MessageId) -> Bool { + if let entry = self.get(id) { + if case .Message = entry { + return true + } + } + + return false + } + + func debugList(peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] { + var list: [HistoryIndexEntry] = [] + self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in + list.append(readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value)) + + return true + }, limit: 0) + return list + } +} diff --git a/Postbox/MessageHistoryTable.swift b/Postbox/MessageHistoryTable.swift new file mode 100644 index 0000000000..9600d5705f --- /dev/null +++ b/Postbox/MessageHistoryTable.swift @@ -0,0 +1,359 @@ +import Foundation + +enum MessageHistoryEntry { + case Msg(Message) + case Hole(MessageHistoryHole) + + var index: MessageIndex { + switch self { + case let .Msg(message): + return MessageIndex(message) + case let .Hole(hole): + return hole.maxIndex + } + } +} + +final class MessageHistoryTable { + let valueBox: ValueBox + let tableId: Int32 + + let messageHistoryIndexTable: MessageHistoryIndexTable + let messageMediaTable: MessageMediaTable + + init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable) { + self.valueBox = valueBox + self.tableId = tableId + self.messageHistoryIndexTable = messageHistoryIndexTable + self.messageMediaTable = messageMediaTable + } + + private func key(index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4)) -> ValueBoxKey { + key.setInt64(0, value: index.id.peerId.toInt64()) + key.setInt32(8, value: index.timestamp) + key.setInt32(8 + 4, value: index.id.namespace) + key.setInt32(8 + 4 + 4, value: index.id.id) + return key + } + + private func lowerBound(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) + key.setInt64(0, value: peerId.toInt64()) + return key + } + + private func upperBound(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) + key.setInt64(0, value: peerId.toInt64()) + return key.successor + } + + private func messagesByPeerId(messages: [StoreMessage]) -> [PeerId: [StoreMessage]] { + var dict: [PeerId: [StoreMessage]] = [:] + + for message in messages { + let peerId = message.id.peerId + if dict[peerId] == nil { + dict[peerId] = [message] + } else { + dict[peerId]!.append(message) + } + } + + return dict + } + + func addMessages(messages: [StoreMessage]) { + let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0)) + let sharedBuffer = WriteBuffer() + let sharedEncoder = Encoder() + + let messagesByPeerId = self.messagesByPeerId(messages) + for (_, peerMessages) in messagesByPeerId { + for message in peerMessages { + if self.messageHistoryIndexTable.messageExists(message.id) { + continue + } + + self.messageHistoryIndexTable.addMessage(MessageIndex(message)) + self.justInsert(message, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder) + } + } + } + + func removeMessages(messageIds: [MessageId]) { + for messageId in messageIds { + if let entry = self.messageHistoryIndexTable.get(messageId) { + if case let .Message(index) = entry { + if let message = self.get(index) { + let embeddedMediaData = message.embeddedMediaData + if embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + self.messageMediaTable.removeEmbeddedMedia(media) + } + embeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in message.referencedMedia { + self.messageMediaTable.removeReference(mediaId) + } + } + + self.messageHistoryIndexTable.removeMessage(messageId) + self.valueBox.remove(self.tableId, key: self.key(index)) + } + } + } + } + + private func justInsert(message: StoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) { + sharedBuffer.reset() + + let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + var length: Int32 = Int32(data.length) + sharedBuffer.write(&length, offset: 0, length: 4) + sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + + var attributeCount: Int32 = Int32(message.attributes.count) + sharedBuffer.write(&attributeCount, offset: 0, length: 4) + for attribute in message.attributes { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(attribute) + let attributeBuffer = sharedEncoder.memoryBuffer() + var attributeBufferLength = Int32(attributeBuffer.length) + sharedBuffer.write(&attributeBufferLength, offset: 0, length: 4) + sharedBuffer.write(attributeBuffer.memory, offset: 0, length: attributeBuffer.length) + } + + var embeddedMedia: [Media] = [] + var referencedMedia: [MediaId] = [] + for media in message.media { + if let mediaId = media.id { + let mediaInsertResult = self.messageMediaTable.set(media, index: MessageIndex(message), messageHistoryTable: self) + switch mediaInsertResult { + case let .Embed(media): + embeddedMedia.append(media) + case .Reference: + referencedMedia.append(mediaId) + } + } else { + embeddedMedia.append(media) + } + } + + var embeddedMediaCount: Int32 = Int32(embeddedMedia.count) + sharedBuffer.write(&embeddedMediaCount, offset: 0, length: 4) + for media in embeddedMedia { + sharedEncoder.reset() + sharedEncoder.encodeRootObject(media) + let mediaBuffer = sharedEncoder.memoryBuffer() + var mediaBufferLength = Int32(mediaBuffer.length) + sharedBuffer.write(&mediaBufferLength, offset: 0, length: 4) + sharedBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) + } + + var referencedMediaCount: Int32 = Int32(referencedMedia.count) + sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4) + for mediaId in referencedMedia { + var idNamespace: Int32 = mediaId.namespace + var idId: Int64 = mediaId.id + sharedBuffer.write(&idNamespace, offset: 0, length: 4) + sharedBuffer.write(&idId, offset: 0, length: 8) + } + + self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer) + } + + func unembedMedia(index: MessageIndex, id: MediaId) -> Media? { + if let message = self.get(index) where message.embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + + let updatedEmbeddedMediaBuffer = WriteBuffer() + var updatedEmbeddedMediaCount = embeddedMediaCount - 1 + updatedEmbeddedMediaBuffer.write(&updatedEmbeddedMediaCount, offset: 0, length: 4) + + var extractedMedia: Media? + + for _ in 0 ..< embeddedMediaCount { + let mediaOffset = message.embeddedMediaData.offset + var mediaLength: Int32 = 0 + var copyMedia = true + message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + + if let mediaId = media.id where mediaId == id { + copyMedia = false + extractedMedia = media + } + } + + if copyMedia { + updatedEmbeddedMediaBuffer.write(message.embeddedMediaData.memory + mediaOffset, offset: 0, length: message.embeddedMediaData.offset - mediaOffset) + } + } + + if let extractedMedia = extractedMedia { + var updatedReferencedMedia = message.referencedMedia + updatedReferencedMedia.append(extractedMedia.id!) + self.storeIntermediateMessage(IntermediateMessage(id: message.id, timestamp: message.timestamp, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index)) + + return extractedMedia + } + } + return nil + } + + func storeIntermediateMessage(message: IntermediateMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer = WriteBuffer()) { + sharedBuffer.reset() + + let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)! + var length: Int32 = Int32(data.length) + sharedBuffer.write(&length, offset: 0, length: 4) + sharedBuffer.write(data.bytes, offset: 0, length: Int(length)) + + sharedBuffer.write(message.attributesData.memory, offset: 0, length: message.attributesData.length) + sharedBuffer.write(message.embeddedMediaData.memory, offset: 0, length: message.embeddedMediaData.length) + + var referencedMediaCount: Int32 = Int32(message.referencedMedia.count) + sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4) + for mediaId in message.referencedMedia { + var idNamespace: Int32 = mediaId.namespace + var idId: Int64 = mediaId.id + sharedBuffer.write(&idNamespace, offset: 0, length: 4) + sharedBuffer.write(&idId, offset: 0, length: 8) + } + + self.valueBox.set(self.tableId, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer) + } + + private func readIntermediateMessage(key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessage { + let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8)) + + var textLength: Int32 = 0 + value.read(&textLength, offset: 0, length: 4) + let text = String(data: NSData(bytes: value.memory + value.offset, length: Int(textLength)), encoding: NSUTF8StringEncoding) ?? "" + value.skip(Int(textLength)) + + let attributesOffset = value.offset + var attributeCount: Int32 = 0 + value.read(&attributeCount, offset: 0, length: 4) + for _ in 0 ..< attributeCount { + var attributeLength: Int32 = 0 + value.read(&attributeLength, offset: 0, length: 4) + value.skip(Int(attributeLength)) + } + let attributesLength = value.offset - attributesOffset + let attributesBytes = malloc(attributesLength) + memcpy(attributesBytes, value.memory + attributesOffset, attributesLength) + let attributesData = ReadBuffer(memory: attributesBytes, length: attributesLength, freeWhenDone: true) + + let embeddedMediaOffset = value.offset + var embeddedMediaCount: Int32 = 0 + value.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + value.read(&mediaLength, offset: 0, length: 4) + value.skip(Int(mediaLength)) + } + let embeddedMediaLength = value.offset - embeddedMediaOffset + let embeddedMediaBytes = malloc(embeddedMediaLength) + memcpy(embeddedMediaBytes, value.memory + embeddedMediaOffset, embeddedMediaLength) + let embeddedMediaData = ReadBuffer(memory: embeddedMediaBytes, length: embeddedMediaLength, freeWhenDone: true) + + var referencedMediaIds: [MediaId] = [] + var referencedMediaIdsCount: Int32 = 0 + value.read(&referencedMediaIdsCount, offset: 0, length: 4) + for _ in 0 ..< referencedMediaIdsCount { + var idNamespace: Int32 = 0 + var idId: Int64 = 0 + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 8) + referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId)) + } + + return IntermediateMessage(id: index.id, timestamp: index.timestamp, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds) + } + + func get(index: MessageIndex) -> IntermediateMessage? { + let key = self.key(index) + if let value = self.valueBox.get(self.tableId, key: key) { + return self.readIntermediateMessage(key, value: value) + } + return nil + } + + func renderMessage(message: IntermediateMessage) -> Message { + var parsedAttributes: [Coding] = [] + var parsedMedia: [Media] = [] + + let attributesData = message.attributesData + if attributesData.length > 4 { + var attributeCount: Int32 = 0 + attributesData.read(&attributeCount, offset: 0, length: 4) + for _ in 0 ..< attributeCount { + var attributeLength: Int32 = 0 + attributesData.read(&attributeLength, offset: 0, length: 4) + if let attribute = Decoder(buffer: MemoryBuffer(memory: attributesData.memory + attributesData.offset, capacity: Int(attributeLength), length: Int(attributeLength), freeWhenDone: false)).decodeRootObject() { + parsedAttributes.append(attribute) + } + attributesData.skip(Int(attributeLength)) + } + } + + let embeddedMediaData = message.embeddedMediaData + if embeddedMediaData.length > 4 { + var embeddedMediaCount: Int32 = 0 + embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4) + for _ in 0 ..< embeddedMediaCount { + var mediaLength: Int32 = 0 + embeddedMediaData.read(&mediaLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media { + parsedMedia.append(media) + } + embeddedMediaData.skip(Int(mediaLength)) + } + } + + for mediaId in message.referencedMedia { + if let media = self.messageMediaTable.get(mediaId) { + parsedMedia.append(media) + } + } + + return Message(id: message.id, timestamp: message.timestamp, text: message.text, attributes: parsedAttributes, media: parsedMedia) + } + + func messagesAround(index: MessageIndex, count: Int) -> [IntermediateMessage] { + var lowerMessages: [IntermediateMessage] = [] + var upperMessages: [IntermediateMessage] = [] + + self.valueBox.range(self.tableId, start: self.key(index), end: self.lowerBound(index.id.peerId), values: { key, value in + lowerMessages.append(self.readIntermediateMessage(key, value: value)) + return true + }, limit: count) + + self.valueBox.range(self.tableId, start: self.key(index).predecessor, end: self.upperBound(index.id.peerId), values: { key, value in + upperMessages.append(self.readIntermediateMessage(key, value: value)) + return true + }, limit: count) + + var messages: [IntermediateMessage] = [] + for message in lowerMessages.reverse() { + messages.append(message) + } + messages.appendContentsOf(upperMessages) + + return messages + } + + func debugList(peerId: PeerId) -> [Message] { + return self.messagesAround(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0), count: 1000).map({self.renderMessage($0)}) + } +} diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift new file mode 100644 index 0000000000..0efad9cde0 --- /dev/null +++ b/Postbox/MessageHistoryView.swift @@ -0,0 +1,341 @@ +import Foundation + +public final class MutableMessageHistoryView: CustomStringConvertible { + public struct RemoveContext { + var invalidEarlier: Bool = false + var invalidLater: Bool = false + var invalidEarlierHole: Bool = false + var invalidLaterHole: Bool = false + var removedMessages: Bool = false + var removedHole: Bool = false + + func empty() -> Bool { + return !self.removedMessages && !invalidEarlier && !invalidLater + } + } + + let count: Int + var earlierMessage: RenderedMessage? + var laterMessage: RenderedMessage? + var messages: [RenderedMessage] + + public init(count: Int, earlierMessage: RenderedMessage?, messages: [RenderedMessage], laterMessage: RenderedMessage?) { + self.count = count + self.earlierMessage = earlierMessage + self.laterMessage = laterMessage + self.messages = messages + } + + public func add(message: RenderedMessage) -> Bool { + if self.messages.count == 0 { + self.messages.append(message) + return true + } else { + let first = MessageIndex(self.messages[self.messages.count - 1].message) + let last = MessageIndex(self.messages[0].message) + + var next: MessageIndex? + if let message = laterMessage { + let messageIndex = MessageIndex(message.message) + next = messageIndex + } + + let index = MessageIndex(message.message) + + if index < last { + let earlierMessage = self.earlierMessage + if earlierMessage == nil || MessageIndex(earlierMessage!.message) < index { + if self.messages.count < self.count { + self.messages.insert(message, atIndex: 0) + } else { + self.earlierMessage = message + } + return true + } else { + return false + } + } else if index > first { + if next != nil && index > next! { + let laterMessage = self.laterMessage + if laterMessage == nil || MessageIndex(laterMessage!.message) > index { + if self.messages.count < self.count { + self.messages.append(message) + } else { + self.laterMessage = message + } + return true + } else { + return false + } + } else { + self.messages.append(message) + if self.messages.count > self.count { + let earliest = self.messages[0] + self.earlierMessage = earliest + self.messages.removeAtIndex(0) + } + return true + } + } else if index != last && index != first { + var i = self.messages.count + while i >= 1 { + if MessageIndex(self.messages[i - 1].message) < index { + break + } + i-- + } + self.messages.insert(message, atIndex: i) + if self.messages.count > self.count { + let earliest = self.messages[0] + self.earlierMessage = earliest + self.messages.removeAtIndex(0) + } + return true + } else { + return false + } + } + } + + public func remove(ids: Set, context: RemoveContext? = nil) -> RemoveContext { + var updatedContext = RemoveContext() + if let context = context { + updatedContext = context + } + + if let earlierMessage = self.earlierMessage where ids.contains(earlierMessage.message.id) { + updatedContext.invalidEarlier = true + } + + if let laterMessage = self.laterMessage where ids.contains(laterMessage.message.id) { + updatedContext.invalidLater = true + } + + if self.messages.count != 0 { + var i = self.messages.count - 1 + while i >= 0 { + if ids.contains(self.messages[i].message.id) { + self.messages.removeAtIndex(i) + updatedContext.removedMessages = true + } + i-- + } + } + + return updatedContext + } + + public func complete(context: RemoveContext, fetchEarlier: (MessageIndex?, Int) -> [RenderedMessage], fetchLater: (MessageIndex?, Int) -> [RenderedMessage]) { + if context.removedMessages { + var addedMessages: [RenderedMessage] = [] + + var latestAnchor: MessageIndex? + if let lastMessage = self.messages.last { + latestAnchor = MessageIndex(lastMessage.message) + } + + if latestAnchor == nil { + if let laterMessage = self.laterMessage { + latestAnchor = MessageIndex(laterMessage.message) + } + } + + if let laterMessage = self.laterMessage { + addedMessages += fetchLater(MessageIndex(laterMessage.message).predecessor(), self.count) + } + if let earlierMessage = self.earlierMessage { + addedMessages += fetchEarlier(MessageIndex(earlierMessage.message).successor(), self.count) + } + + addedMessages += self.messages + addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) + var i = addedMessages.count - 1 + while i >= 1 { + if addedMessages[i].message.id == addedMessages[i - 1].message.id { + addedMessages.removeAtIndex(i) + } + i-- + } + self.messages = [] + + var anchorIndex = addedMessages.count - 1 + if let latestAnchor = latestAnchor { + var i = addedMessages.count - 1 + while i >= 0 { + if MessageIndex(addedMessages[i].message) <= latestAnchor { + anchorIndex = i + break + } + i-- + } + } + + self.laterMessage = nil + if anchorIndex + 1 < addedMessages.count { + self.laterMessage = addedMessages[anchorIndex + 1] + } + + i = anchorIndex + while i >= 0 && i > anchorIndex - self.count { + self.messages.insert(addedMessages[i], atIndex: 0) + i-- + } + + self.earlierMessage = nil + if anchorIndex - self.count >= 0 { + self.earlierMessage = addedMessages[anchorIndex - self.count] + } + } + else { + if context.invalidEarlier { + var earlyId: MessageIndex? + let i = 0 + if i < self.messages.count { + earlyId = MessageIndex(self.messages[i].message) + } + + let earlierMessages = fetchEarlier(earlyId, 1) + self.earlierMessage = earlierMessages.first + } + + if context.invalidLater { + var laterId: MessageIndex? + let i = self.messages.count - 1 + if i >= 0 { + laterId = MessageIndex(self.messages[i].message) + } + + let laterMessages = fetchLater(laterId, 1) + self.laterMessage = laterMessages.first + } + } + } + + + public func incompleteMessages() -> [Message] { + var result: [Message] = [] + + if let earlierMessage = self.earlierMessage where earlierMessage.incomplete { + result.append(earlierMessage.message) + } + + if let laterMessage = self.laterMessage where laterMessage.incomplete { + result.append(laterMessage.message) + } + + for message in self.messages { + if message.incomplete { + result.append(message.message) + } + } + + return result + } + + public func completeMessages(messages: [MessageId : RenderedMessage]) { + if let earlierMessage = self.earlierMessage { + if let renderedMessage = messages[earlierMessage.message.id] { + self.earlierMessage = renderedMessage + } + } + + if let laterMessage = self.laterMessage { + if let renderedMessage = messages[laterMessage.message.id] { + self.laterMessage = renderedMessage + } + } + + var i = 0 + while i < self.messages.count { + if let message = messages[self.messages[i].message.id] { + self.messages[i] = message + } + i++ + } + } + + public var description: String { + var string = "" + string += "...(" + if let value = self.earlierMessage { + string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)" + } + string += ") —— " + + string += "[" + var first = true + for message in self.messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" + } + string += "]" + + string += " —— (" + if let value = self.laterMessage { + string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)" + } + string += ")..." + + return string + } +} + +public final class MessageHistoryView: CustomStringConvertible { + public let hasEarlier: Bool + private let earlierId: MessageIndex? + public let hasLater: Bool + private let laterId: MessageIndex? + public let messages: [RenderedMessage] + + init(_ mutableView: MutableMessageHistoryView) { + self.hasEarlier = mutableView.earlierMessage != nil + self.hasLater = mutableView.laterMessage != nil + self.messages = mutableView.messages + + if let earlierMessage = mutableView.earlierMessage { + self.earlierId = MessageIndex(earlierMessage.message) + } else { + self.earlierId = nil + } + + if let laterMessage = mutableView.laterMessage { + self.laterId = MessageIndex(laterMessage.message) + } else { + self.laterId = nil + } + } + + public var description: String { + var string = "" + if self.hasEarlier { + string += "more(" + if let earlierId = self.earlierId { + string += "\(earlierId.id.namespace): \(earlierId.id.id)—\(earlierId.timestamp)" + } + string += ") " + } + string += "[" + var first = true + for message in self.messages { + if first { + first = false + } else { + string += ", " + } + string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" + } + string += "]" + if self.hasLater { + string += " more(" + if let laterId = self.laterId { + string += "\(laterId.id.namespace): \(laterId.id.id)—\(laterId.timestamp)" + } + string += ")" + } + return string + } +} diff --git a/Postbox/MessageMediaTable.swift b/Postbox/MessageMediaTable.swift new file mode 100644 index 0000000000..0fd6794974 --- /dev/null +++ b/Postbox/MessageMediaTable.swift @@ -0,0 +1,210 @@ +import Foundation + +private enum MediaEntryType: Int8 { + case Direct + case MessageReference +} + +enum InsertMediaResult { + case Reference + case Embed(Media) +} + +enum DebugMediaEntry { + case Direct(Media, Int) + case MessageReference(MessageIndex) +} + +final class MessageMediaTable { + let valueBox: ValueBox + let tableId: Int32 + + let mediaCleanupTable: MediaCleanupTable + + init(valueBox: ValueBox, tableId: Int32, mediaCleanupTable: MediaCleanupTable) { + self.valueBox = valueBox + self.tableId = tableId + self.mediaCleanupTable = mediaCleanupTable + } + + func key(id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey { + key.setInt32(0, value: id.namespace) + key.setInt64(4, value: id.id) + return key + } + + func get(id: MediaId) -> Media? { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { + return media + } + } + } + return nil + } + + func set(media: Media, index: MessageIndex, messageHistoryTable: MessageHistoryTable, sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) -> InsertMediaResult { + if let id = media.id { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + value.skip(Int(dataLength)) + + sharedWriteBuffer.reset() + sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset) + + var messageReferenceCount: Int32 = 0 + value.read(&messageReferenceCount, offset: 0, length: 4) + messageReferenceCount++ + sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) + + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + + return .Reference + } else if type == MediaEntryType.MessageReference.rawValue { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + + let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) + if referencedMessageIndex == index { + return .Embed(media) + } + + if let media = messageHistoryTable.unembedMedia(referencedMessageIndex, id: id) { + sharedWriteBuffer.reset() + var directType: Int8 = MediaEntryType.Direct.rawValue + sharedWriteBuffer.write(&directType, offset: 0, length: 1) + + sharedEncoder.reset() + sharedEncoder.encodeRootObject(media) + let mediaBuffer = sharedEncoder.memoryBuffer() + var mediaBufferLength = Int32(mediaBuffer.length) + sharedWriteBuffer.write(&mediaBufferLength, offset: 0, length: 4) + sharedWriteBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length) + + var messageReferenceCount: Int32 = 2 + sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) + + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + } + + return .Reference + } else { + return .Embed(media) + } + } else { + sharedWriteBuffer.reset() + var type: Int8 = MediaEntryType.MessageReference.rawValue + sharedWriteBuffer.write(&type, offset: 0, length: 1) + var idPeerId: Int64 = index.id.peerId.toInt64() + var idNamespace: Int32 = index.id.namespace + var idId: Int32 = index.id.id + var idTimestamp: Int32 = index.timestamp + sharedWriteBuffer.write(&idPeerId, offset: 0, length: 8) + sharedWriteBuffer.write(&idNamespace, offset: 0, length: 4) + sharedWriteBuffer.write(&idId, offset: 0, length: 4) + sharedWriteBuffer.write(&idTimestamp, offset: 0, length: 4) + + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + + return .Embed(media) + } + } else { + return .Embed(media) + } + } + + func removeReference(id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) { + if let value = self.valueBox.get(self.tableId, key: self.key(id)) { + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + let mediaOffset = value.offset + value.skip(Int(dataLength)) + + sharedWriteBuffer.reset() + sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset) + + var messageReferenceCount: Int32 = 0 + value.read(&messageReferenceCount, offset: 0, length: 4) + messageReferenceCount -= 1 + sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4) + + if messageReferenceCount <= 0 { + if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + mediaOffset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { + self.mediaCleanupTable.add(media) + } + self.valueBox.remove(self.tableId, key: self.key(id)) + } else { + self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy()) + } + } else if type == MediaEntryType.MessageReference.rawValue { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + } + } + + func removeEmbeddedMedia(media: Media) { + if let id = media.id { + self.valueBox.remove(self.tableId, key: self.key(id)) + } + self.mediaCleanupTable.add(media) + } + + func debugList() -> [DebugMediaEntry] { + var entries: [DebugMediaEntry] = [] + + let upperBoundKey = ValueBoxKey(length: 8 + 4) + memset(upperBoundKey.memory, 0xff, 8 + 4) + self.valueBox.range(self.tableId, start: ValueBoxKey(length: 0), end: upperBoundKey, values: { key, value in + var type: Int8 = 0 + value.read(&type, offset: 0, length: 1) + if type == MediaEntryType.Direct.rawValue { + var dataLength: Int32 = 0 + value.read(&dataLength, offset: 0, length: 4) + if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media { + + value.skip(Int(dataLength)) + + var messageReferenceCount: Int32 = 0 + value.read(&messageReferenceCount, offset: 0, length: 4) + + entries.append(.Direct(media, Int(messageReferenceCount))) + } + } else if type == MediaEntryType.MessageReference.rawValue { + var idPeerId: Int64 = 0 + var idNamespace: Int32 = 0 + var idId: Int32 = 0 + var idTimestamp: Int32 = 0 + value.read(&idPeerId, offset: 0, length: 8) + value.read(&idNamespace, offset: 0, length: 4) + value.read(&idId, offset: 0, length: 4) + value.read(&idTimestamp, offset: 0, length: 4) + + let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp) + + entries.append(.MessageReference(referencedMessageIndex)) + } + + return true + }, limit: 1000) + + return entries + } +} \ No newline at end of file diff --git a/Postbox/MessageView.swift b/Postbox/MessageView.swift deleted file mode 100644 index aea71cd29b..0000000000 --- a/Postbox/MessageView.swift +++ /dev/null @@ -1,428 +0,0 @@ -import Foundation - -public final class MutableMessageView: CustomStringConvertible { - public struct RemoveContext { - var invalidEarlier: Set - var invalidLater: Set - var removedMessages: Bool - - init() { - self.invalidEarlier = [] - self.invalidLater = [] - self.removedMessages = false - } - - func empty() -> Bool { - return !self.removedMessages && self.invalidEarlier.count == 0 && self.invalidLater.count == 0 - } - } - - let namespaces: [MessageId.Namespace] - let count: Int - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - var later: [MessageId.Namespace : RenderedMessage] = [:] - var messages: [RenderedMessage] - - public init(namespaces: [MessageId.Namespace], count: Int, earlier: [MessageId.Namespace : RenderedMessage], messages: [RenderedMessage], later: [MessageId.Namespace : RenderedMessage]) { - self.namespaces = namespaces - self.count = count - self.earlier = earlier - self.later = later - self.messages = messages - } - - public func add(message: RenderedMessage) -> Bool { - if self.messages.count == 0 { - self.messages.append(message) - return true - } else { - let first = MessageIndex(self.messages[self.messages.count - 1].message) - let last = MessageIndex(self.messages[0].message) - - var next: MessageIndex? - for namespace in self.namespaces { - if let message = later[namespace] { - let messageIndex = MessageIndex(message.message) - if next == nil || messageIndex < next! { - next = messageIndex - } - } - } - - let index = MessageIndex(message.message) - - if index < last { - let earlierMessage = self.earlier[message.message.id.namespace] - if earlierMessage == nil || earlierMessage!.message.id.id < message.message.id.id { - if self.messages.count < self.count { - self.messages.insert(message, atIndex: 0) - } else { - self.earlier[message.message.id.namespace] = message - } - return true - } else { - return false - } - } else if index > first { - if next != nil && index > next! { - let laterMessage = self.later[message.message.id.namespace] - if laterMessage == nil || laterMessage!.message.id.id > message.message.id.id { - if self.messages.count < self.count { - self.messages.append(message) - } else { - self.later[message.message.id.namespace] = message - } - return true - } else { - return false - } - } else { - self.messages.append(message) - if self.messages.count > self.count { - let earliest = self.messages[0] - self.earlier[earliest.message.id.namespace] = earliest - self.messages.removeAtIndex(0) - } - return true - } - } else if index != last && index != first { - var i = self.messages.count - while i >= 1 { - if MessageIndex(self.messages[i - 1].message) < index { - break - } - i-- - } - self.messages.insert(message, atIndex: i) - if self.messages.count > self.count { - let earliest = self.messages[0] - self.earlier[earliest.message.id.namespace] = earliest - self.messages.removeAtIndex(0) - } - return true - } else { - return false - } - } - } - - public func remove(ids: Set, context: RemoveContext? = nil) -> RemoveContext { - var updatedContext = RemoveContext() - if let context = context { - updatedContext = context - } - - for (_, message) in self.earlier { - if ids.contains(message.message.id) { - updatedContext.invalidEarlier.insert(message.message.id.namespace) - } - } - - for (_, message) in self.later { - if ids.contains(message.message.id) { - updatedContext.invalidLater.insert(message.message.id.namespace) - } - } - - if self.messages.count != 0 { - var i = self.messages.count - 1 - while i >= 0 { - if ids.contains(self.messages[i].message.id) { - self.messages.removeAtIndex(i) - updatedContext.removedMessages = true - } - i-- - } - } - - return updatedContext - } - - public func complete(context: RemoveContext, fetchEarlier: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage], fetchLater: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage]) { - if context.removedMessages { - var addedMessages: [RenderedMessage] = [] - - var latestAnchor: MessageIndex? - if let lastMessage = self.messages.last { - latestAnchor = MessageIndex(lastMessage.message) - } - - if latestAnchor == nil { - for (_, message) in self.later { - let messageIndex = MessageIndex(message.message) - if latestAnchor == nil || latestAnchor! > messageIndex { - latestAnchor = messageIndex - } - } - } - - for namespace in self.namespaces { - if let later = self.later[namespace] { - addedMessages += fetchLater(namespace, later.message.id.id - 1, self.count) - } - if let earlier = self.earlier[namespace] { - addedMessages += fetchEarlier(namespace, earlier.message.id.id + 1, self.count) - } - } - - addedMessages += self.messages - addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - var i = addedMessages.count - 1 - while i >= 1 { - if addedMessages[i].message.id == addedMessages[i - 1].message.id { - addedMessages.removeAtIndex(i) - } - i-- - } - self.messages = [] - - var anchorIndex = addedMessages.count - 1 - if let latestAnchor = latestAnchor { - var i = addedMessages.count - 1 - while i >= 0 { - if MessageIndex(addedMessages[i].message) <= latestAnchor { - anchorIndex = i - break - } - i-- - } - } - - self.later.removeAll(keepCapacity: true) - if anchorIndex + 1 < addedMessages.count { - for namespace in self.namespaces { - var i = anchorIndex + 1 - while i < addedMessages.count { - if addedMessages[i].message.id.namespace == namespace { - self.later[namespace] = addedMessages[i] - break - } - i++ - } - } - } - - i = anchorIndex - while i >= 0 && i > anchorIndex - self.count { - self.messages.insert(addedMessages[i], atIndex: 0) - i-- - } - - self.earlier.removeAll(keepCapacity: true) - if anchorIndex - self.count >= 0 { - for namespace in self.namespaces { - i = anchorIndex - self.count - while i >= 0 { - if addedMessages[i].message.id.namespace == namespace { - self.earlier[namespace] = addedMessages[i] - break - } - i-- - } - } - } - } - else { - for namespace in context.invalidEarlier { - var earlyId: MessageId.Id? - var i = 0 - while i < self.messages.count { - if self.messages[i].message.id.namespace == namespace { - earlyId = self.messages[i].message.id.id - break - } - i++ - } - - let earlierMessages = fetchEarlier(namespace, earlyId, 1) - if earlierMessages.count == 0 { - self.earlier.removeValueForKey(namespace) - } else { - self.earlier[namespace] = earlierMessages[0] - } - } - - for namespace in context.invalidLater { - var lateId: MessageId.Id? - var i = self.messages.count - 1 - while i >= 0 { - if self.messages[i].message.id.namespace == namespace { - lateId = self.messages[i].message.id.id - break - } - i-- - } - - let laterMessages = fetchLater(namespace, lateId, 1) - if laterMessages.count == 0 { - self.later.removeValueForKey(namespace) - } else { - self.later[namespace] = laterMessages[0] - } - } - } - } - - - public func incompleteMessages() -> [Message] { - var result: [Message] = [] - - for (_, message) in self.earlier { - if message.incomplete { - result.append(message.message) - } - } - for (_, message) in self.later { - if message.incomplete { - result.append(message.message) - } - } - - for message in self.messages { - if message.incomplete { - result.append(message.message) - } - } - - return result - } - - public func completeMessages(messages: [MessageId : RenderedMessage]) { - var earlier = self.earlier - for (namespace, message) in self.earlier { - if let message = messages[message.message.id] { - earlier[namespace] = message - } - } - self.earlier = earlier - - var later = self.later - for (namespace, message) in self.later { - if let message = messages[message.message.id] { - later[namespace] = message - } - } - self.later = later - - var i = 0 - while i < self.messages.count { - if let message = messages[self.messages[i].message.id] { - self.messages[i] = message - } - i++ - } - } - - public var description: String { - var string = "" - string += "...(" - var first = true - for namespace in self.namespaces { - if let value = self.earlier[namespace] { - if first { - first = false - } else { - string += ", " - } - string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)" - } - } - string += ") —— " - - string += "[" - first = true - for message in self.messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" - } - string += "]" - - string += " —— (" - first = true - for namespace in self.namespaces { - if let value = self.later[namespace] { - if first { - first = false - } else { - string += ", " - } - string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)" - } - } - string += ")..." - - return string - } -} - -public final class MessageView: CustomStringConvertible { - public let hasEarlier: Bool - private let earlierIds: [MessageIndex] - public let hasLater: Bool - private let laterIds: [MessageIndex] - public let messages: [RenderedMessage] - - init(_ mutableView: MutableMessageView) { - self.hasEarlier = mutableView.earlier.count != 0 - self.hasLater = mutableView.later.count != 0 - self.messages = mutableView.messages - - var earlierIds: [MessageIndex] = [] - for (_, message) in mutableView.earlier { - earlierIds.append(MessageIndex(message.message)) - } - self.earlierIds = earlierIds - - var laterIds: [MessageIndex] = [] - for (_, message) in mutableView.later { - laterIds.append(MessageIndex(message.message)) - } - self.laterIds = laterIds - } - - public var description: String { - var string = "" - if self.hasEarlier { - string += "more(" - var first = true - for id in self.earlierIds { - if first { - first = false - } else { - string += ", " - } - string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)" - } - string += ") " - } - string += "[" - var first = true - for message in self.messages { - if first { - first = false - } else { - string += ", " - } - string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)" - } - string += "]" - if self.hasLater { - string += " more(" - var first = true - for id in self.laterIds { - if first { - first = false - } else { - string += ", " - } - string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)" - } - string += ")" - } - return string - } -} diff --git a/Postbox/Peer.swift b/Postbox/Peer.swift index a6f7fac52c..04bf16f2d9 100644 --- a/Postbox/Peer.swift +++ b/Postbox/Peer.swift @@ -59,11 +59,13 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable { } public init(_ buffer: ReadBuffer) { - self.namespace = 0 - self.id = 0 - memcpy(&self.namespace, buffer.memory, 4) - memcpy(&self.id, buffer.memory + 4, 4) + var namespace: Int32 = 0 + var id: Int32 = 0 + memcpy(&namespace, buffer.memory, 4) + self.namespace = namespace + memcpy(&id, buffer.memory + 4, 4) + self.id = id } public func encodeToBuffer(buffer: WriteBuffer) { diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index e5a8ed5528..9ddd12ec63 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -48,13 +48,13 @@ public final class Modifier { public final class Postbox { private let basePath: String private let messageNamespaces: [MessageId.Namespace] - private let absoluteIndexedMessageNamespaces: [MessageId.Namespace] + private let absoluteIndexedMessageNamespace: MessageId.Namespace private let queue = Queue(name: "org.telegram.postbox.Postbox") private var valueBox: ValueBox! - private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe)>] = [:] - private var deferredMessageViewsToUpdate: [(MutableMessageView, Pipe)] = [] + private var peerMessageHistoryViews: [PeerId : Bag<(MutableMessageHistoryView, Pipe)>] = [:] + private var deferredMessageHistoryViewsToUpdate: [(MutableMessageHistoryView, Pipe)] = [] private var peerViews: Bag<(MutablePeerView, Pipe)> = Bag() private var deferredPeerViewsToUpdate: [(MutablePeerView, Pipe)] = [] private var peerPipes: [PeerId : Pipe] = [:] @@ -63,10 +63,14 @@ public final class Postbox { public let mediaBox: MediaBox - public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespaces: [MessageId.Namespace]) { + public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespace: MessageId.Namespace?) { self.basePath = basePath self.messageNamespaces = messageNamespaces - self.absoluteIndexedMessageNamespaces = absoluteIndexedMessageNamespaces + if let absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace { + self.absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace + } else { + self.absoluteIndexedMessageNamespace = MessageId.Namespace.max + } self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() } @@ -83,7 +87,7 @@ public final class Postbox { self.valueBox = SqliteValueBox(basePath: self.basePath + "/db") //self.valueBox = LmdbValueBox(basePath: self.basePath + "/db") var userVersion: Int32 = 0 - let currentUserVersion: Int32 = 3 + let currentUserVersion: Int32 = 4 if let value = self.valueBox.get(Table_Meta.id, key: Table_Meta.key()) { value.read(&userVersion, offset: 0, length: 4) @@ -166,123 +170,11 @@ public final class Postbox { } private func addMessages(messages: [Message], medias: [Media]) { - let encoder = Encoder() - for (peerId, peerMessages) in messagesGroupedByPeerId(messages) { - var maxMessage: (MessageIndex, Message)? - - var messageIds: [MessageId] = [] - var seenMessageIds = Set() - for message in peerMessages { - if !seenMessageIds.contains(message.id) { - seenMessageIds.insert(message.id) - messageIds.append(message.id) - } - } - - var existingMessageIds = Set() - - let existingMessageKey = Table_Message.emptyKey() - existingMessageKey.setInt64(0, value: peerId.toInt64()) - for id in messageIds { - if self.valueBox.exists(Table_Message.id, key: Table_Message.key(id, key: existingMessageKey)) { - existingMessageIds.insert(id) - } - } - - var addedMessages: [Message] = [] - - let mediaMessageIdKey = Table_Media_MessageIds.emptyKey() - for message in peerMessages { - if existingMessageIds.contains(message.id) { - continue - } - existingMessageIds.insert(message.id) - addedMessages.append(message) - - let index = MessageIndex(message) - if maxMessage == nil || index > maxMessage!.0 { - maxMessage = (index, message) - } - - encoder.reset() - encoder.encodeRootObject(message) - - for id in message.mediaIds { - self.valueBox.set(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(id, messageId: message.id, key: mediaMessageIdKey), value: MemoryBuffer()) - } - - self.valueBox.set(Table_Message.id, key: Table_Message.key(message.id), value: Table_Message.set(message)) - - let absoluteKey = Table_AbsoluteMessageId.emptyKey() - if self.absoluteIndexedMessageNamespaces.contains(message.id.namespace) { - self.valueBox.set(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(message.id.id, key: absoluteKey), value: Table_AbsoluteMessageId.set(message.id)) - } - } - - if let relatedViews = self.peerMessageViews[peerId] { - for record in relatedViews.copyItems() { - var updated = false - for message in addedMessages { - if record.0.add(RenderedMessage(message: message)) { - updated = true - } - } - - if updated { - self.deferMessageViewUpdate(record.0, pipe: record.1) - } - } - } - - if let maxMessage = maxMessage { - self.updatePeerEntry(peerId, message: RenderedMessage(message: maxMessage.1)) - } - } - - var existingMediaIds = Set() - - let existingMediaKey = Table_Media.emptyKey() - for media in medias { - if let id = media.id { - if self.valueBox.exists(Table_Media.id, key: Table_Media.key(id, key: existingMediaKey)) { - existingMediaIds.insert(id) - } - } - } - - let mediaKey = Table_Media.emptyKey() - for media in medias { - if let id = media.id { - if existingMediaIds.contains(id) { - continue - } - existingMediaIds.insert(id) - - self.valueBox.set(Table_Media.id, key: Table_Media.key(id, key: mediaKey), value: Table_Media.set(media)) - } - } } private func mediaWithIds(ids: [MediaId]) -> [MediaId : Media] { - if ids.count == 0 { - return [:] - } else { - var result: [MediaId : Media] = [:] - - let mediaKey = Table_Media.emptyKey() - for id in ids { - if let value = self.valueBox.get(Table_Media.id, key: Table_Media.key(id, key: mediaKey)) { - if let media = Table_Media.get(value) { - result[id] = media - } else { - print("can't parse media") - } - } - } - - return result - } + return [:] } var cachedPeers: [PeerId : Peer] = [:] @@ -338,19 +230,19 @@ public final class Postbox { } } - private func deferMessageViewUpdate(view: MutableMessageView, pipe: Pipe) { + private func deferMessageHistoryViewUpdate(view: MutableMessageHistoryView, pipe: Pipe) { var i = 0 var found = false - while i < self.deferredMessageViewsToUpdate.count { - if self.deferredMessageViewsToUpdate[i].1 === pipe { - self.deferredMessageViewsToUpdate[i] = (view, pipe) + while i < self.deferredMessageHistoryViewsToUpdate.count { + if self.deferredMessageHistoryViewsToUpdate[i].1 === pipe { + self.deferredMessageHistoryViewsToUpdate[i] = (view, pipe) found = true break } i++ } if !found { - self.deferredMessageViewsToUpdate.append((view, pipe)) + self.deferredMessageHistoryViewsToUpdate.append((view, pipe)) } } @@ -371,10 +263,10 @@ public final class Postbox { entry.1.putNext(PeerView(entry.0)) } - let deferredMessageViewsToUpdate = self.deferredMessageViewsToUpdate - self.deferredMessageViewsToUpdate.removeAll() + let deferredMessageHistoryViewsToUpdate = self.deferredMessageHistoryViewsToUpdate + self.deferredMessageHistoryViewsToUpdate.removeAll() - for entry in deferredMessageViewsToUpdate { + for entry in deferredMessageHistoryViewsToUpdate { let viewRenderedMessages = self.renderedMessages(entry.0.incompleteMessages()) if viewRenderedMessages.count != 0 { var viewRenderedMessagesDict: [MessageId : RenderedMessage] = [:] @@ -384,7 +276,7 @@ public final class Postbox { entry.0.completeMessages(viewRenderedMessagesDict) } - entry.1.putNext(MessageView(entry.0)) + entry.1.putNext(MessageHistoryView(entry.0)) } } @@ -458,10 +350,10 @@ public final class Postbox { var result: [MessageId] = [] - let key = Table_AbsoluteMessageId.emptyKey() + let key = Table_GlobalMessageId.emptyKey() for id in ids { - if let value = self.valueBox.get(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(id, key: key)) { - result.append(Table_AbsoluteMessageId.get(id, value: value)) + if let value = self.valueBox.get(Table_GlobalMessageId.id, key: Table_GlobalMessageId.key(id, key: key)) { + result.append(Table_GlobalMessageId.get(id, value: value)) } } @@ -469,55 +361,7 @@ public final class Postbox { } private func deleteMessagesWithIds(ids: [MessageId]) { - for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) { - if let relatedViews = self.peerMessageViews[peerId] { - for (view, pipe) in relatedViews.copyItems() { - let context = view.remove(Set(messageIds)) - if !context.empty() { - view.complete(context, fetchEarlier: self.fetchMessagesRelative(peerId, earlier: true), fetchLater: self.fetchMessagesRelative(peerId, earlier: false)) - self.deferMessageViewUpdate(view, pipe: pipe) - } - } - } - } - var touchedMediaIds = Set() - - let removeMediaMessageIdKey = Table_Media_MessageIds.emptyKey() - for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) { - let messageKey = Table_Message.emptyKey() - for id in messageIds { - if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(id, key: messageKey)) { - if let message = Table_Message.get(value) { - for mediaId in message.mediaIds { - touchedMediaIds.insert(mediaId) - self.valueBox.remove(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(mediaId, messageId: message.id, key: removeMediaMessageIdKey)) - } - } - } - } - - for id in messageIds { - self.valueBox.remove(Table_Message.id, key: Table_Message.key(id, key: messageKey)) - } - - for mediaId in touchedMediaIds { - var referenced = false - self.valueBox.range(Table_Media_MessageIds.id, start: Table_Media_MessageIds.lowerBoundKey(mediaId), end: Table_Media_MessageIds.upperBoundKey(mediaId), keys: { key in - referenced = true - return false - }, limit: 1) - - if !referenced { - //TODO write to cleanup queue - self.valueBox.remove(Table_Media.id, key: Table_Media.key(mediaId)) - } - } - - let tail = self.fetchMessagesTail(peerId, count: 1) - - self.updatePeerEntry(peerId, message: tail.first, replace: true) - } } private func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) { @@ -572,19 +416,21 @@ public final class Postbox { return Signal { subscriber in self.queue.dispatch { //#if DEBUG - let startTime = CFAbsoluteTimeGetCurrent() + //let startTime = CFAbsoluteTimeGetCurrent() //#endif + //self.valueBox.beginStats() self.valueBox.begin() let result = f(Modifier(postbox: self)) - //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") //#if DEBUG //startTime = CFAbsoluteTimeGetCurrent() //#endif self.valueBox.commit() //#if DEBUG - print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)") + //self.valueBox.endStats() //#endif self.performDeferredUpdates() @@ -596,282 +442,6 @@ public final class Postbox { } } - private func findAdjacentMessageIds(peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex) -> (MessageId.Id?, MessageId.Id?) { - /*var minId: MessageId.Id? - var maxId: MessageId.Id? - - let lowerBoundKey = ValueBoxKey(length: 8 + 4) - lowerBoundKey.setInt64(0, value: peerId.toInt64()) - lowerBoundKey.setInt32(8, value: namespace) - - let upperBoundKey = ValueBoxKey(length: 8 + 4) - upperBoundKey.setInt64(0, value: peerId.toInt64()) - upperBoundKey.setInt32(8, value: namespace) - - self.valueBox.range("peer_messages", start: lowerBoundKey, end: upperBoundKey.successor, keys: { key in - - }, limit: 1) - - for row in self.database.prepareCached("SELECT MIN(id), MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?").run(peerId.toInt64(), Int64(namespace)) { - minId = MessageId.Id(row[0] as! Int64) - maxId = MessageId.Id(row[1] as! Int64) - } - - if let minId = minId, maxId = maxId { - var minTimestamp: Int32! - var maxTimestamp: Int32! - for row in self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (?, ?)").run(peerId.toInt64(), Int64(namespace), Int64(minId), Int64(maxId)) { - let id = Int32(row[0] as! Int64) - let timestamp = Int32(row[1] as! Int64) - if id == minId { - minTimestamp = timestamp - } else { - maxTimestamp = timestamp - } - } - - let earlierMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id <= ? LIMIT 1") - let laterMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id >= ? LIMIT 1") - - func lowerBound(timestamp: Int32) -> MessageId.Id? { - var leftId = minId - var leftTimestamp = minTimestamp - var rightId = maxId - var rightTimestamp = maxTimestamp - - while leftTimestamp <= timestamp && rightTimestamp >= timestamp { - let approximateMiddleId = leftId + (rightId - leftId) / 2 - if approximateMiddleId == leftId { - return rightId - } - var middleId: MessageId.Id? - var middleTimestamp: Int32? - for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) { - middleId = MessageId.Id(row[0] as! Int64) - middleTimestamp = Int32(row[1] as! Int64) - break - } - if middleId == leftId { - return rightId - } - - if let middleId = middleId, middleTimestamp = middleTimestamp { - if middleTimestamp >= timestamp { - rightId = middleId - rightTimestamp = middleTimestamp - } else { - leftId = middleId - leftTimestamp = middleTimestamp - } - } else { - return nil - } - } - - return leftId - } - - func upperBound(timestamp: Int32) -> MessageId.Id? { - var leftId = minId - var leftTimestamp = minTimestamp - var rightId = maxId - var rightTimestamp = maxTimestamp - - while leftTimestamp <= timestamp && rightTimestamp >= timestamp { - let approximateMiddleId = leftId + (rightId - leftId) / 2 - if approximateMiddleId == leftId { - return leftId - } - var middleId: MessageId.Id? - var middleTimestamp: Int32? - for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) { - middleId = MessageId.Id(row[0] as! Int64) - middleTimestamp = Int32(row[1] as! Int64) - break - } - if middleId == leftId { - return leftId - } - - if let middleId = middleId, middleTimestamp = middleTimestamp { - if middleTimestamp <= timestamp { - rightId = middleId - rightTimestamp = middleTimestamp - } else { - leftId = middleId - leftTimestamp = middleTimestamp - } - } else { - return nil - } - } - - return rightTimestamp - } - - if index.id.namespace < namespace { - let left = upperBound(index.timestamp - 1) - let right = lowerBound(index.timestamp) - return (left, right) - } else { - let left = upperBound(index.timestamp) - let right = lowerBound(index.timestamp + 1) - return (left, right) - } - } else { - return (nil, nil) - }*/ - - return (nil, nil) - } - - private func fetchMessagesAround(peerId: PeerId, anchorId: MessageId, count: Int) -> ([RenderedMessage], [MessageId.Namespace : RenderedMessage], [MessageId.Namespace : RenderedMessage]) { - var messages: [RenderedMessage] = [] - - messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: anchorId.namespace, id: anchorId.id, count: count + 1) - messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: anchorId.namespace, id: anchorId.id - 1, count: count + 1) - - messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - var i = messages.count - 1 - while i >= 1 { - if messages[i].message.id == messages[i - 1].message.id { - messages.removeAtIndex(i) - } - i-- - } - - if messages.count == 0 { - return ([], [:], [:]) - } else { - var index: MessageIndex! - for message in messages { - if message.message.id == anchorId { - index = MessageIndex(message.message) - break - } - } - if index == nil { - var closestId: MessageId.Id = messages[0].message.id.id - var closestDistance = abs(closestId - anchorId.id) - let closestTimestamp: Int32 = messages[0].message.timestamp - for message in messages { - if abs(message.message.id.id - anchorId.id) < closestDistance { - closestId = message.message.id.id - closestDistance = abs(message.message.id.id - anchorId.id) - } - } - index = MessageIndex(id: MessageId(peerId: peerId, namespace: anchorId.namespace, id: closestId), timestamp: closestTimestamp) - } - - for namespace in self.messageNamespaces { - if namespace != anchorId.namespace { - let (left, right) = self.findAdjacentMessageIds(peerId, namespace: namespace, index: index) - if let left = left { - messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: left + 1, count: count + 1) - } - if let right = right { - messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: namespace, id: right - 1, count: count + 1) - } - } - } - - messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - var i = messages.count - 1 - while i >= 1 { - if messages[i].message.id == messages[i - 1].message.id { - messages.removeAtIndex(i) - } - i-- - } - - var anchorIndex = messages.count / 2 - i = 0 - while i < messages.count { - if messages[i].message.id == index.id { - anchorIndex = i - break - } - i++ - } - - var filteredMessages: [RenderedMessage] = [] - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - var later: [MessageId.Namespace : RenderedMessage] = [:] - - i = anchorIndex - var j = anchorIndex - 1 - var leftIndex = j - var rightIndex = i - - while i < messages.count || j >= 0 { - if i < messages.count && filteredMessages.count < count { - filteredMessages.append(messages[i]) - rightIndex = i - } - if j >= 0 && filteredMessages.count < count { - filteredMessages.append(messages[j]) - leftIndex = j - } - - i++ - j-- - } - - i = leftIndex - 1 - while i >= 0 { - if earlier[messages[i].message.id.namespace] == nil { - earlier[messages[i].message.id.namespace] = messages[i] - } - i-- - } - - i = rightIndex + 1 - while i < messages.count { - if later[messages[i].message.id.namespace] == nil { - later[messages[i].message.id.namespace] = messages[i] - } - i++ - } - - filteredMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) }) - - return (filteredMessages, earlier, later) - } - } - - private func fetchMessagesRelative(peerId: PeerId, earlier: Bool)(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [RenderedMessage] { - var messages: [Message] = [] - - let lowerBound = Table_Message.lowerBoundKey(peerId, namespace: namespace) - let upperBound = Table_Message.upperBoundKey(peerId, namespace: namespace) - - let bound: ValueBoxKey - if let id = id { - bound = Table_Message.key(MessageId(peerId: peerId, namespace: namespace, id: id)) - } else if earlier { - bound = upperBound - } else { - bound = lowerBound - } - - let values: (ValueBoxKey, ReadBuffer) -> Bool = { _, value in - if let message = Table_Message.get(value) { - messages.append(message) - } else { - print("can't parse message") - } - return true - } - - if earlier { - self.valueBox.range(Table_Message.id, start: bound, end: lowerBound, values: values, limit: count) - } else { - self.valueBox.range(Table_Message.id, start: bound, end: upperBound, values: values, limit: count) - } - - return self.renderedMessages(messages) - } - private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] { var entries: [PeerViewEntryIndex] = [] @@ -918,7 +488,7 @@ public final class Postbox { } var message: Message? - if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex.id)) { + if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex)) { message = Table_Message.get(value) } @@ -943,123 +513,38 @@ public final class Postbox { } } - entries.sortInPlace({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) + //entries.sortInPlace() - return entries + return entries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) }) } private func renderedMessages(messages: [Message]) -> [RenderedMessage] { - if messages.count == 0 { - return [] - } - - var peerIds = Set() - var mediaIds = Set() - - for message in messages { - for peerId in message.peerIds { - peerIds.insert(peerId) - } - for mediaId in message.mediaIds { - mediaIds.insert(mediaId) - } - } - - var arrayPeerIds: [PeerId] = [] - for id in peerIds { - arrayPeerIds.append(id) - } - let peers = self.peersWithIds(arrayPeerIds) - - var arrayMediaIds: [MediaId] = [] - for id in mediaIds { - arrayMediaIds.append(id) - } - let medias = self.mediaWithIds(arrayMediaIds) - - var result: [RenderedMessage] = [] - - for message in messages { - if message.peerIds.count == 0 && message.mediaIds.count == 0 { - result.append(RenderedMessage(message: message, peers: [], media: [])) - } else { - var messagePeers: [Peer] = [] - for id in message.peerIds { - if let peer = peers[id] { - messagePeers.append(peer) - } - } - - var messageMedia: [Media] = [] - for id in message.mediaIds { - if let media = medias[id] { - messageMedia.append(media) - } - } - - result.append(RenderedMessage(message: message, peers: messagePeers, media: messageMedia)) - } - } - - return result + return [] } - private func fetchMessagesTail(peerId: PeerId, count: Int) -> [RenderedMessage] { - var messages: [RenderedMessage] = [] - - for namespace in self.messageNamespaces { - messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: nil, count: count) - } - - messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message)}) - - return messages - } - - public func tailMessageViewForPeerId(peerId: PeerId, count: Int) -> Signal { + public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int) -> Signal { return Signal { subscriber in let startTime = CFAbsoluteTimeGetCurrent() let disposable = MetaDisposable() self.queue.dispatch { - let tail = self.fetchMessagesTail(peerId, count: count + 1) - print("tailMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") + print("tailMessageHistoryViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - var messages: [RenderedMessage] = [] - var i = tail.count - 1 - while i >= 0 && i >= tail.count - count { - messages.insert(tail[i], atIndex: 0) - i-- - } + let mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil) + let record = (mutableView, Pipe()) - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - - for namespace in self.messageNamespaces { - var i = tail.count - count - 1 - while i >= 0 { - if tail[i].message.id.namespace == namespace { - earlier[namespace] = tail[i] - break - } - i-- - } - } - - let mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:]) - let record = (mutableView, Pipe()) - - let index: Bag<(MutableMessageView, Pipe)>.Index - if let bag = self.peerMessageViews[peerId] { + let index: Bag<(MutableMessageHistoryView, Pipe)>.Index + if let bag = self.peerMessageHistoryViews[peerId] { index = bag.add(record) } else { - let bag = Bag<(MutableMessageView, Pipe)>() + let bag = Bag<(MutableMessageHistoryView, Pipe)>() index = bag.add(record) - self.peerMessageViews[peerId] = bag + self.peerMessageHistoryViews[peerId] = bag } - subscriber.putNext(MessageView(mutableView)) + subscriber.putNext(MessageHistoryView(mutableView)) let pipeDisposable = record.1.signal().start(next: { next in subscriber.putNext(next) @@ -1070,7 +555,7 @@ public final class Postbox { if let strongSelf = self { strongSelf.queue.dispatch { - if let bag = strongSelf.peerMessageViews[peerId] { + if let bag = strongSelf.peerMessageHistoryViews[peerId] { bag.remove(index) } } @@ -1083,58 +568,31 @@ public final class Postbox { } } - public func aroundMessageViewForPeerId(peerId: PeerId, id: MessageId, count: Int) -> Signal { + public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() self.queue.dispatch { - let mutableView: MutableMessageView + let mutableView: MutableMessageHistoryView let startTime = CFAbsoluteTimeGetCurrent() - let around = self.fetchMessagesAround(peerId, anchorId: id, count: count) - if around.0.count == 0 { - let tail = self.fetchMessagesTail(peerId, count: count + 1) - - var messages: [RenderedMessage] = [] - var i = tail.count - 1 - while i >= 0 && i >= tail.count - count { - messages.insert(tail[i], atIndex: 0) - i-- - } - - var earlier: [MessageId.Namespace : RenderedMessage] = [:] - - for namespace in self.messageNamespaces { - var i = tail.count - count - 1 - while i >= 0 { - if tail[i].message.id.namespace == namespace { - earlier[namespace] = tail[i] - break - } - i-- - } - } - - mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:]) - } else { - mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: around.1, messages: around.0, later: around.2) - } + mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil) print("aroundMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") - let record = (mutableView, Pipe()) + let record = (mutableView, Pipe()) - let index: Bag<(MutableMessageView, Pipe)>.Index - if let bag = self.peerMessageViews[peerId] { + let index: Bag<(MutableMessageHistoryView, Pipe)>.Index + if let bag = self.peerMessageHistoryViews[peerId] { index = bag.add(record) } else { - let bag = Bag<(MutableMessageView, Pipe)>() + let bag = Bag<(MutableMessageHistoryView, Pipe)>() index = bag.add(record) - self.peerMessageViews[peerId] = bag + self.peerMessageHistoryViews[peerId] = bag } - subscriber.putNext(MessageView(mutableView)) + subscriber.putNext(MessageHistoryView(mutableView)) let pipeDisposable = record.1.signal().start(next: { next in subscriber.putNext(next) @@ -1145,7 +603,7 @@ public final class Postbox { if let strongSelf = self { strongSelf.queue.dispatch { - if let bag = strongSelf.peerMessageViews[peerId] { + if let bag = strongSelf.peerMessageHistoryViews[peerId] { bag.remove(index) } } diff --git a/Postbox/PostboxTables.swift b/Postbox/PostboxTables.swift index c17f466265..8f048b3912 100644 --- a/Postbox/PostboxTables.swift +++ b/Postbox/PostboxTables.swift @@ -41,33 +41,31 @@ struct Table_Message { static let id: Int32 = 4 static func emptyKey() -> ValueBoxKey { - return ValueBoxKey(length: 8 + 4 + 4) + return ValueBoxKey(length: 8 + 4 + 4 + 4) } - static func lowerBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey { - let key = ValueBoxKey(length: 8 + 4) + static func lowerBoundKey(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) key.setInt64(0, value: peerId.toInt64()) - key.setInt32(8, value: namespace) return key } - static func upperBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey { - let key = ValueBoxKey(length: 8 + 4) + static func upperBoundKey(peerId: PeerId) -> ValueBoxKey { + let key = ValueBoxKey(length: 8) key.setInt64(0, value: peerId.toInt64()) - key.setInt32(8, value: namespace) return key.successor } - static func key(messageId: MessageId, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey { - key.setInt64(0, value: messageId.peerId.toInt64()) - key.setInt32(8, value: messageId.namespace) - key.setInt32(8 + 4, value: messageId.id) + static func key(index: MessageIndex, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey { + key.setInt64(0, value: index.id.peerId.toInt64()) + key.setInt32(8, value: index.timestamp) + key.setInt32(8 + 4, value: index.id.namespace) + key.setInt32(8 + 4 + 4, value: index.id.id) return key } static func set(message: Message, encoder: Encoder = Encoder()) -> MemoryBuffer { encoder.reset() - encoder.encodeRootObject(message) return encoder.memoryBuffer() } @@ -79,14 +77,14 @@ struct Table_Message { } } -struct Table_AbsoluteMessageId { +struct Table_GlobalMessageId { static let id: Int32 = 5 static func emptyKey() -> ValueBoxKey { return ValueBoxKey(length: 4) } - static func key(id: Int32, key: ValueBoxKey = Table_AbsoluteMessageId.emptyKey()) -> ValueBoxKey { + static func key(id: Int32, key: ValueBoxKey = Table_GlobalMessageId.emptyKey()) -> ValueBoxKey { key.setInt32(0, value: id) return key } @@ -272,3 +270,31 @@ struct Table_PeerEntry { return index } } + +struct Table_MessageIndex { + static let id: Int32 = 11 + + static func emptyKey() -> ValueBoxKey { + return ValueBoxKey(length: 8 + 4 + 4) + } + + static func key(id: MessageId, key: ValueBoxKey = Table_MessageIndex.emptyKey()) -> ValueBoxKey { + key.setInt64(0, value: id.peerId.toInt64()) + key.setInt32(8, value: id.namespace) + key.setInt32(8 + 4, value: id.id) + return key + } + + static func set(index: MessageIndex) -> MemoryBuffer { + let buffer = WriteBuffer() + var timestamp = index.timestamp + buffer.write(×tamp, offset: 0, length: 4) + return buffer.readBufferNoCopy() + } + + static func get(id: MessageId, value: ReadBuffer) -> MessageIndex { + var timestamp: Int32 = 0 + value.read(×tamp, offset: 0, length: 4) + return MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id), timestamp: timestamp) + } +} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 4eb9b4d862..e5a954326f 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -65,6 +65,10 @@ public final class SqliteValueBox: ValueBox { private var insertStatements: [Int32 : SqlitePreparedStatement] = [:] private var deleteStatements: [Int32 : SqlitePreparedStatement] = [:] + private var readQueryTime: CFAbsoluteTime = 0.0 + private var writeQueryTime: CFAbsoluteTime = 0.0 + private var commitTime: CFAbsoluteTime = 0.0 + public init(basePath: String) { do { try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil) @@ -91,12 +95,28 @@ public final class SqliteValueBox: ValueBox { } } + deinit { + self.clearStatements() + } + + public func beginStats() { + self.readQueryTime = 0.0 + self.writeQueryTime = 0.0 + self.commitTime = 0.0 + } + + public func endStats() { + print("(SqliteValueBox stats read: \(self.readQueryTime * 1000.0) ms, write: \(self.writeQueryTime * 1000.0) ms, commit: \(self.commitTime * 1000.0) ms") + } + public func begin() { self.database.transaction() } public func commit() { + let startTime = CFAbsoluteTimeGetCurrent() self.database.commit() + self.commitTime += CFAbsoluteTimeGetCurrent() - startTime } private func getStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement { @@ -372,6 +392,7 @@ public final class SqliteValueBox: ValueBox { } public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? { + let startTime = CFAbsoluteTimeGetCurrent() if self.tables.contains(table) { let statement = self.getStatement(table, key: key) @@ -384,6 +405,8 @@ public final class SqliteValueBox: ValueBox { statement.reset() + self.readQueryTime += CFAbsoluteTimeGetCurrent() - startTime + return buffer } @@ -405,6 +428,8 @@ public final class SqliteValueBox: ValueBox { if self.tables.contains(table) { let statement: SqlitePreparedStatement + var startTime = CFAbsoluteTimeGetCurrent() + if start < end { if limit <= 0 { statement = self.rangeValueAscStatementNoLimit(table, start: start, end: end) @@ -419,10 +444,20 @@ public final class SqliteValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + let key = statement.keyAt(0) let value = statement.valueAt(1) + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + if !values(key, value) { break } @@ -436,6 +471,8 @@ public final class SqliteValueBox: ValueBox { if self.tables.contains(table) { let statement: SqlitePreparedStatement + var startTime = CFAbsoluteTimeGetCurrent() + if start < end { if limit <= 0 { statement = self.rangeKeyAscStatementNoLimit(table, start: start, end: end) @@ -450,9 +487,19 @@ public final class SqliteValueBox: ValueBox { } } + var currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + + startTime = currentTime + while statement.step() { + startTime = CFAbsoluteTimeGetCurrent() + let key = statement.keyAt(0) + currentTime = CFAbsoluteTimeGetCurrent() + self.readQueryTime += currentTime - startTime + if !keys(key) { break } @@ -470,6 +517,8 @@ public final class SqliteValueBox: ValueBox { self.database.execute("INSERT INTO __meta_tables(name) VALUES (\(table))") } + let startTime = CFAbsoluteTimeGetCurrent() + var exists = false let existsStatement = self.existsStatement(table, key: key) if existsStatement.step() { @@ -488,19 +537,25 @@ public final class SqliteValueBox: ValueBox { } statement.reset() } + + self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } public func remove(table: Int32, key: ValueBoxKey) { if self.tables.contains(table) { + let startTime = CFAbsoluteTimeGetCurrent() + let statement = self.deleteStatement(table, key: key) while statement.step() { } statement.reset() + + self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime } } - public func drop() { + private func clearStatements() { for (_, statement) in self.getStatements { statement.destroy() } @@ -565,6 +620,10 @@ public final class SqliteValueBox: ValueBox { statement.destroy() } self.deleteStatements.removeAll() + } + + public func drop() { + self.clearStatements() for table in self.tables { self.database.execute("DROP TABLE IF EXISTS t\(table)") diff --git a/Postbox/ValueBox.swift b/Postbox/ValueBox.swift index 7549d15ea1..83973eeb8f 100644 --- a/Postbox/ValueBox.swift +++ b/Postbox/ValueBox.swift @@ -4,6 +4,9 @@ public protocol ValueBox { func begin() func commit() + func beginStats() + func endStats() + func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int) func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int) func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? diff --git a/PostboxTests/CodingTests.swift b/PostboxTests/CodingTests.swift index f8f70bdaec..00c75ab465 100644 --- a/PostboxTests/CodingTests.swift +++ b/PostboxTests/CodingTests.swift @@ -165,45 +165,6 @@ class SerializationTests: XCTestCase { XCTAssert(decoder.decodeInt32ForKey("a") == 12345, "int32 failed") } - func testIndexBaselinePerformance() { - let basePath = "/tmp/postboxtest" - do { - try NSFileManager.defaultManager().removeItemAtPath(basePath) - } catch _ { } - let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) - postbox._prepareBaselineIndexPerformance() - - measureBlock { - postbox._measureBaselineIndexPerformance() - } - } - - func testBlobReadPerformance() { - let basePath = "/tmp/postboxtest" - do { - try NSFileManager.defaultManager().removeItemAtPath(basePath) - } catch _ { } - let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) - postbox._prepareBlobIndexPerformance() - - measureBlock { - postbox._measureBlobReadPerformance() - } - } - - func testBlobIndexPerformance() { - let basePath = "/tmp/postboxtest" - do { - try NSFileManager.defaultManager().removeItemAtPath(basePath) - } catch _ { } - let postbox = Postbox(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: []) - postbox._prepareBlobIndexPerformance() - - measureBlock { - postbox._measureBlobIndexPerformance() - } - } - func testKeys() { let key1 = ValueBoxKey(length: 8) let key2 = ValueBoxKey(length: 8) @@ -230,7 +191,7 @@ class SerializationTests: XCTestCase { } func testKeyValue() { - let basePath = "/tmp/postboxtest" + /*let basePath = "/tmp/postboxtest" do { try NSFileManager.defaultManager().removeItemAtPath(basePath) } catch _ { } @@ -256,6 +217,6 @@ class SerializationTests: XCTestCase { return true }, limit: 10) } - } + }*/ } } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift new file mode 100644 index 0000000000..9c742da934 --- /dev/null +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -0,0 +1,427 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +private let peerId = PeerId(namespace: 1, id: 1) +private let namespace: Int32 = 1 + +private enum Item: Equatable, CustomStringConvertible { + case Message(Int32, Int32) + case Hole(Int32, Int32, Int32) + + init(_ item: HistoryIndexEntry) { + switch item { + case let .Message(index): + self = .Message(index.id.id, index.timestamp) + case let .Hole(hole): + self = .Hole(hole.min, hole.maxIndex.id.id, hole.maxIndex.timestamp) + } + } + + var description: String { + switch self { + case let .Message(id, timestamp): + return "Message(\(id), \(timestamp))" + case let .Hole(minId, maxId, maxTimestamp): + return "Hole(\(minId), \(maxId), \(maxTimestamp))" + } + } +} + +private func ==(lhs: Item, rhs: Item) -> Bool { + switch lhs { + case let .Message(id, timestamp): + switch rhs { + case let .Message(rId, rTimestamp): + return id == rId && timestamp == rTimestamp + case .Hole: + return false + } + case let .Hole(minId, maxId, maxTimestamp): + switch rhs { + case .Message: + return false + case let .Hole(rMinId, rMaxId, rMaxTimestamp): + return minId == rMinId && maxId == rMaxId && maxTimestamp == rMaxTimestamp + } + } +} + +@testable import Postbox + +class MessageHistoryIndexTableTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var indexTable: MessageHistoryIndexTable? + + override func setUp() { + super.setUp() + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + self.valueBox = SqliteValueBox(basePath: path!) + + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1) + } + + override func tearDown() { + super.tearDown() + + self.indexTable = nil + + self.valueBox = nil + let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + self.path = nil + } + + func addHole(id: Int32) { + self.indexTable!.addHole(MessageId(peerId: peerId, namespace: namespace, id: id)) + } + + func addMessage(id: Int32, _ timestamp: Int32) { + self.indexTable!.addMessage(MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp)) + } + + func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32)]) { + self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, indices: messages.map({MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1)})) + } + + func removeMessage(id: Int32) { + self.indexTable!.removeMessage(MessageId(peerId: peerId, namespace: namespace, id: id)) + } + + private func expect(items: [Item]) { + let actualItems = self.indexTable!.debugList(peerId, namespace: namespace).map { return Item($0) } + if items != actualItems { + XCTFail("Expected\n\(items)\nGot\n\(actualItems)") + } + } + + func testEmpty() { + expect([]) + } + + func testAddMessageToEmpty() { + addMessage(100, 100) + expect([.Message(100, 100)]) + + addMessage(110, 110) + expect([.Message(100, 100), .Message(110, 110)]) + + addMessage(90, 90) + expect([.Message(90, 90), .Message(100, 100), .Message(110, 110)]) + } + + func testAddHoleToEmpty() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testAddHoleToFullHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addHole(110) + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testAddMessageToFullHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageDividingUpperHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(100, 100) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, 99, 100), .Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testAddMessageDividingLowerHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(80, 80) + expect([.Hole(1, 79, 80), .Message(80, 80), .Hole(81, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingUpperHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(91, 91) + expect([.Hole(1, 89, 90), .Message(90, 90), .Message(91, 91), .Hole(92, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingLowerHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(90, 90) + expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + addMessage(89, 89) + expect([.Hole(1, 88, 89), .Message(89, 89), .Message(90, 90), .Hole(91, Int32.max, Int32.max)]) + } + + func testAddMessageOffsettingLeftmostHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(1, 1) + + expect([.Message(1, 1), .Hole(2, Int32.max, Int32.max)]) + } + + func testAddMessageRemovingLefmostHole() { + addHole(100) + expect([.Hole(1, Int32.max, Int32.max)]) + + addMessage(2, 2) + expect([.Hole(1, 1, 2), .Message(2, 2), .Hole(3, Int32.max, Int32.max)]) + + addMessage(1, 1) + expect([.Message(1, 1), .Message(2, 2), .Hole(3, Int32.max, Int32.max)]) + } + + func testAddHoleLowerThanMessage() { + addMessage(100, 100) + addHole(1) + + expect([.Hole(1, 99, 100), .Message(100, 100)]) + } + + func testAddHoleHigherThanMessage() { + addMessage(100, 100) + addHole(200) + + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testIgnoreHigherHole() { + addHole(200) + expect([.Hole(1, Int32.max, Int32.max)]) + addHole(400) + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testIgnoreHigherHoleAfterMessage() { + addMessage(100, 100) + addHole(200) + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + addHole(400) + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testAddHoleBetweenMessages() { + addMessage(100, 100) + addMessage(200, 200) + addHole(150) + + expect([.Message(100, 100), .Hole(101, 199, 200), .Message(200, 200)]) + } + + func testFillHoleEmpty() { + fillHole(1, .Complete, []) + expect([]) + } + + func testFillHoleComplete() { + addHole(100) + + fillHole(1, .Complete, [(100, 100), (200, 200)]) + expect([.Message(100, 100), .Message(200, 200)]) + } + + func testFillHoleUpperToLowerPartial() { + addHole(100) + + fillHole(1, .UpperToLower, [(100, 100), (200, 200)]) + expect([.Hole(1, 99, 100), .Message(100, 100), .Message(200, 200)]) + } + + func testFillHoleUpperToLowerToBounds() { + addHole(100) + + fillHole(1, .UpperToLower, [(1, 1), (200, 200)]) + expect([.Message(1, 1), .Message(200, 200)]) + } + + func testFillHoleLowerToUpperToBounds() { + addHole(100) + + fillHole(1, .LowerToUpper, [(100, 100), (Int32.max, 200)]) + expect([.Message(100, 100), .Message(Int32.max, 200)]) + } + + func testFillHoleLowerToUpperPartial() { + addHole(100) + + fillHole(1, .LowerToUpper, [(100, 100), (200, 200)]) + expect([.Message(100, 100), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesUpperToLower() { + addHole(1) + + addMessage(100, 100) + addMessage(200, 200) + + fillHole(199, .UpperToLower, [(150, 150)]) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesLowerToUpper() { + addHole(1) + + addMessage(100, 100) + addMessage(200, 200) + + fillHole(199, .LowerToUpper, [(150, 150)]) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Hole(151, 199, 200), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesComplete() { + addHole(1) + + addMessage(100, 100) + addMessage(200, 200) + + fillHole(199, .Complete, [(150, 150)]) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)]) + } + + func testFillHoleBetweenMessagesWithMessage() { + addMessage(200, 200) + addMessage(202, 202) + addHole(201) + addMessage(201, 201) + + expect([.Message(200, 200), .Message(201, 201), .Message(202, 202)]) + } + + func testFillHoleWithNoMessagesComplete() { + addMessage(100, 100) + addHole(1) + + fillHole(99, .Complete, []) + + expect([.Message(100, 100)]) + } + + func testFillHoleIgnoreOverMessage() { + addMessage(100, 100) + addMessage(101, 101) + + fillHole(100, .Complete, [(90, 90)]) + + expect([.Message(90, 90), .Message(100, 100), .Message(101, 101)]) + } + + func testFillHoleWithOverflow() { + addMessage(100, 100) + addMessage(200, 200) + addHole(150) + + fillHole(199, .UpperToLower, [(150, 150), (300, 300)]) + + expect([.Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Message(300, 300)]) + } + + func testIgnoreHoleOverMessageBetweenMessages() { + addMessage(199, 199) + addMessage(200, 200) + addHole(200) + + expect([.Message(199, 199), .Message(200, 200)]) + } + + func testMergeHoleAfterDeletingMessage() { + addMessage(100, 100) + addHole(1) + addHole(200) + + expect([.Hole(1, 99, 100), .Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + + removeMessage(100) + + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testMergeHoleLowerAfterDeletingMessage() { + addMessage(100, 100) + addHole(1) + addMessage(200, 200) + + removeMessage(100) + + expect([.Hole(1, 199, 200), .Message(200, 200)]) + } + + func testMergeHoleUpperAfterDeletingMessage() { + addMessage(100, 100) + addMessage(200, 200) + addHole(300) + + removeMessage(200) + + expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)]) + } + + func testExtendLowerHoleAfterDeletingMessage() { + addMessage(100, 100) + addHole(100) + + removeMessage(100) + + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testExtendUpperHoleAfterDeletingMessage() { + addMessage(100, 100) + addHole(101) + + removeMessage(100) + + expect([.Hole(1, Int32.max, Int32.max)]) + } + + func testDeleteMessageBelowMessage() { + addMessage(100, 100) + addMessage(200, 200) + removeMessage(100) + + expect([.Message(200, 200)]) + } + + func testDeleteMessageAboveMessage() { + addMessage(100, 100) + addMessage(200, 200) + removeMessage(200) + + expect([.Message(100, 100)]) + } + + func testDeleteMessageBetweenMessages() { + addMessage(100, 100) + addMessage(200, 200) + addMessage(300, 300) + removeMessage(200) + + expect([.Message(100, 100), .Message(300, 300)]) + } +} diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift new file mode 100644 index 0000000000..925f1acb8c --- /dev/null +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -0,0 +1,306 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +private let peerId = PeerId(namespace: 1, id: 1) +private let namespace: Int32 = 1 + +private func ==(lhs: (Int32, Int32, String, [Media]), rhs: (Int32, Int32, String, [Media])) -> Bool { + if lhs.3.count != rhs.3.count { + return false + } + for i in 0 ..< lhs.3.count { + if !lhs.3[i].isEqual(rhs.3[i]) { + return false + } + } + return lhs.0 == rhs.0 && lhs.1 == rhs.1 && lhs.2 == rhs.2 +} + +private class TestEmbeddedMedia: Media, CustomStringConvertible { + var id: MediaId? { return nil } + let data: String + + init(data: String) { + self.data = data + } + + required init(decoder: Decoder) { + self.data = decoder.decodeStringForKey("s") + } + + func encode(encoder: Encoder) { + encoder.encodeString(self.data, forKey: "s") + } + + func isEqual(other: Media) -> Bool { + if let other = other as? TestEmbeddedMedia { + return self.data == other.data + } + return false + } + + var description: String { + return "TestEmbeddedMedia(\(self.data))" + } +} + +private class TestExternalMedia: Media { + let id: MediaId? + let data: String + + init(id: Int64, data: String) { + self.id = MediaId(namespace: namespace, id: id) + self.data = data + } + + required init(decoder: Decoder) { + self.id = MediaId(namespace: decoder.decodeInt32ForKey("i.n"), id: decoder.decodeInt64ForKey("i.i")) + self.data = decoder.decodeStringForKey("s") + } + + func encode(encoder: Encoder) { + encoder.encodeInt32(self.id!.namespace, forKey: "i.n") + encoder.encodeInt64(self.id!.id, forKey: "i.i") + encoder.encodeString(self.data, forKey: "s") + } + + func isEqual(other: Media) -> Bool { + if let other = other as? TestExternalMedia { + return self.id == other.id && self.data == other.data + } + return false + } + + var description: String { + return "TestExternalMedia(\(self.id!.id), \(self.data))" + } +} + +private enum MediaEntry: Equatable { + case Direct(Media, Int) + case MessageReference(Int32) + + init(_ entry: DebugMediaEntry) { + switch entry { + case let .Direct(media, referenceCount): + self = .Direct(media, referenceCount) + case let .MessageReference(index): + self = MessageReference(index.id.id) + } + } +} + +private func ==(lhs: MediaEntry, rhs: MediaEntry) -> Bool { + switch lhs { + case let .Direct(lhsMedia, lhsReferenceCount): + switch rhs { + case let .Direct(rhsMedia, rhsReferenceCount): + return lhsMedia.isEqual(rhsMedia) && lhsReferenceCount == rhsReferenceCount + case .MessageReference: + return false + } + case let .MessageReference(lhsId): + switch rhs { + case .Direct: + return false + case let .MessageReference(rhsId): + return lhsId == rhsId + } + } +} + +class MessageHistoryTableTests: XCTestCase { + var valueBox: ValueBox? + var path: String? + + var indexTable: MessageHistoryIndexTable? + var mediaTable: MessageMediaTable? + var mediaCleanupTable: MediaCleanupTable? + var historyTable: MessageHistoryTable? + + override class func setUp() { + super.setUp() + + declareEncodable(TestEmbeddedMedia.self, f: {TestEmbeddedMedia(decoder: $0)}) + declareEncodable(TestExternalMedia.self, f: {TestExternalMedia(decoder: $0)}) + } + + override func setUp() { + super.setUp() + + var randomId: Int64 = 0 + arc4random_buf(&randomId, 8) + path = NSTemporaryDirectory().stringByAppendingString("\(randomId)") + self.valueBox = SqliteValueBox(basePath: path!) + + self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1) + self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3) + self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!) + self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!) + } + + override func tearDown() { + super.tearDown() + + self.historyTable = nil + self.indexTable = nil + self.mediaTable = nil + self.mediaCleanupTable = nil + + self.valueBox = nil + let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!) + self.path = nil + } + + private func addMessage(id: Int32, _ timestamp: Int32, _ text: String, _ media: [Media] = []) { + self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, text: text, attributes: [], media: media)]) + } + + private func removeMessages(ids: [Int32]) { + self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) })) + } + + func expectMessages(messages: [(Int32, Int32, String, [Media])]) { + let actualMessages = self.historyTable!.debugList(peerId).map({ ($0.id.id, $0.timestamp, $0.text, $0.media) }) + var equal = true + if messages.count != actualMessages.count { + equal = false + } else { + for i in 0 ..< messages.count { + if !(messages[i] == actualMessages[i]) { + equal = false + break + } + } + } + + if !equal { + XCTFail("Expected\n\(messages)\nActual\n\(actualMessages)") + } + } + + private func expectMedia(media: [MediaEntry]) { + let actualMedia = self.mediaTable!.debugList().map({MediaEntry($0)}) + if media != actualMedia { + XCTFail("Expected\n\(media)\nActual\n\(actualMedia)") + } + } + + private func expectCleanupMedia(media: [Media]) { + let actualMedia = self.mediaCleanupTable!.debugList() + var equal = true + if media.count != actualMedia.count { + equal = false + } else { + for i in 0 ..< media.count { + if !media[i].isEqual(actualMedia[i]) { + equal = false + break + } + } + } + + if !equal { + XCTFail("Expected\n\(media)\nActual\n\(actualMedia)") + } + } + + func testInsertMessageIntoEmpty() { + addMessage(100, 100, "t100") + addMessage(200, 200, "t200") + + expectMessages([(100, 100, "t100", []), (200, 200, "t200", [])]) + } + + func testInsertMessageIgnoreOverwrite() { + addMessage(100, 100, "t100") + addMessage(100, 200, "t200") + + expectMessages([(100, 100, "t100", [])]) + } + + func testInsertMessageWithEmbeddedMedia() { + addMessage(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")]) + + expectMessages([(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")])]) + expectMedia([]) + } + + func testInsertMessageWithExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + + expectMessages([(100, 100, "t100", [media])]) + expectMedia([.MessageReference(100)]) + } + + func testUnembedExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + addMessage(200, 200, "t200", [media]) + + expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])]) + expectMedia([.Direct(media, 2)]) + } + + func testIgnoreOverrideExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + let media1 = TestExternalMedia(id: 10, data: "abc2") + addMessage(100, 100, "t100", [media]) + addMessage(200, 200, "t200", [media1]) + + expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])]) + expectMedia([.Direct(media, 2)]) + } + + func testRemoveSingleMessage() { + addMessage(100, 100, "t100", []) + + removeMessages([100]) + + expectMessages([]) + expectMedia([]) + } + + func testRemoveMessageWithEmbeddedMedia() { + let media = TestEmbeddedMedia(data: "abc1") + addMessage(100, 100, "t100", [media]) + self.removeMessages([100]) + + expectMessages([]) + expectMedia([]) + expectCleanupMedia([media]) + } + + func testRemoveOnlyReferenceToExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + removeMessages([100]) + + expectMessages([]) + expectMedia([]) + expectCleanupMedia([media]) + } + + func testRemoveReferenceToExternalMedia() { + let media = TestExternalMedia(id: 10, data: "abc1") + addMessage(100, 100, "t100", [media]) + addMessage(200, 200, "t200", [media]) + removeMessages([100]) + + expectMessages([(200, 200, "t200", [media])]) + expectMedia([.Direct(media, 1)]) + expectCleanupMedia([]) + + removeMessages([200]) + + expectMessages([]) + expectMedia([]) + expectCleanupMedia([media]) + } +} \ No newline at end of file