no message

Postbox.xcodeproj/project.pbxproj

@@ -7,7 +7,7 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
-		D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageView.swift */; };
+		D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; };
 		D00E0FB31B84CEDA002E4EB5 /* Display.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D00E0FB21B84CEDA002E4EB5 /* Display.framework */; };
 		D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; };
 		D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; };
@@ -29,11 +29,15 @@
 		D07516771B2EC90400AE42E0 /* fts3_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */; };
 		D07516781B2EC90400AE42E0 /* SQLite-Bridging.h in Headers */ = {isa = PBXBuildFile; fileRef = D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */; };
 		D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */ = {isa = PBXBuildFile; fileRef = D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */; };
+		D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C71391C501F0700779C0F /* MessageHistoryHole.swift */; };
+		D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */; };
+		D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */; };
 		D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; };
 		D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; };
 		D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; };
 		D0977FA21B82930C009994B2 /* PostboxCodingUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */; };
 		D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977FA31B829F87009994B2 /* PostboxTables.swift */; };
+		D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; };
 		D0B76BE71B66639F0095CF45 /* DeferredString.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0B76BE61B66639F0095CF45 /* DeferredString.swift */; };
 		D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; };
 		D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D224ED1B4D6ABD0085E26D /* Functions.swift */; };
@@ -45,6 +49,9 @@
 		D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7871B28AE9C00A402D9 /* Coding.swift */; };
 		D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A79D1B28B50400A402D9 /* Message.swift */; };
 		D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E3A7A11B28B7DC00A402D9 /* Media.swift */; };
+		D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */; };
+		D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */; };
+		D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -58,7 +65,7 @@
 /* End PBXContainerItemProxy section */
 
 /* Begin PBXFileReference section */
-		D003E4E51B38DBDB00C22CBC /* MessageView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageView.swift; sourceTree = "<group>"; };
+		D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryView.swift; sourceTree = "<group>"; };
 		D00E0FB21B84CEDA002E4EB5 /* Display.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Display.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/Display.framework"; sourceTree = "<group>"; };
 		D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; };
 		D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; };
@@ -81,11 +88,15 @@
 		D07516741B2EC90400AE42E0 /* fts3_tokenizer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = fts3_tokenizer.h; sourceTree = "<group>"; };
 		D07516751B2EC90400AE42E0 /* SQLite-Bridging.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "SQLite-Bridging.h"; sourceTree = "<group>"; };
 		D07516761B2EC90400AE42E0 /* SQLite-Bridging.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "SQLite-Bridging.m"; sourceTree = "<group>"; };
+		D08C71391C501F0700779C0F /* MessageHistoryHole.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryHole.swift; sourceTree = "<group>"; };
+		D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTable.swift; sourceTree = "<group>"; };
+		D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTable.swift; sourceTree = "<group>"; };
 		D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = "<group>"; };
 		D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = "<group>"; };
 		D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = "<group>"; };
 		D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxCodingUtils.swift; sourceTree = "<group>"; };
 		D0977FA31B829F87009994B2 /* PostboxTables.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PostboxTables.swift; sourceTree = "<group>"; };
+		D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = "<group>"; };
 		D0B76BE61B66639F0095CF45 /* DeferredString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeferredString.swift; sourceTree = "<group>"; };
 		D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftSignalKit.framework; path = "../../../../Library/Developer/Xcode/DerivedData/Telegram-iOS-gbpsmqzuwcmmxadrqcwyrluaftwp/Build/Products/Debug-iphoneos/SwiftSignalKit.framework"; sourceTree = "<group>"; };
 		D0D224ED1B4D6ABD0085E26D /* Functions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = Functions.swift; path = submodules/sqlite.swift/SQLite/Functions.swift; sourceTree = SOURCE_ROOT; };
@@ -100,6 +111,9 @@
 		D0E3A7871B28AE9C00A402D9 /* Coding.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Coding.swift; sourceTree = "<group>"; };
 		D0E3A79D1B28B50400A402D9 /* Message.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Message.swift; sourceTree = "<group>"; };
 		D0E3A7A11B28B7DC00A402D9 /* Media.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Media.swift; sourceTree = "<group>"; };
+		D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageMediaTable.swift; sourceTree = "<group>"; };
+		D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryTableTests.swift; sourceTree = "<group>"; };
+		D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaCleanupTable.swift; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -207,8 +221,9 @@
 				D0B76BE61B66639F0095CF45 /* DeferredString.swift */,
 				D0E3A7831B28AE0900A402D9 /* Peer.swift */,
 				D0E3A79D1B28B50400A402D9 /* Message.swift */,
+				D08C71391C501F0700779C0F /* MessageHistoryHole.swift */,
 				D0E3A7A11B28B7DC00A402D9 /* Media.swift */,
-				D003E4E51B38DBDB00C22CBC /* MessageView.swift */,
+				D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */,
 				D0D225251B4D84930085E26D /* PeerView.swift */,
 				D0977FA11B82930C009994B2 /* PostboxCodingUtils.swift */,
 				D0E3A7811B28ADD000A402D9 /* Postbox.swift */,
@@ -218,6 +233,10 @@
 				D0977F9B1B822DB4009994B2 /* ValueBox.swift */,
 				D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */,
 				D00E0FBD1B85D1B5002E4EB5 /* LmdbValueBox.swift */,
+				D08C713B1C51283C00779C0F /* MessageHistoryIndexTable.swift */,
+				D08C713D1C512EA500779C0F /* MessageHistoryTable.swift */,
+				D0F9E85A1C565EBB00037222 /* MessageMediaTable.swift */,
+				D0F9E8621C579F0200037222 /* MediaCleanupTable.swift */,
 				D0E3A74D1B28A7E300A402D9 /* Supporting Files */,
 			);
 			path = Postbox;
@@ -239,6 +258,8 @@
 			children = (
 				D0E3A75A1B28A7E300A402D9 /* Supporting Files */,
 				D044E15D1B2ACB9C001EE087 /* CodingTests.swift */,
+				D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */,
+				D0F9E8601C57766A00037222 /* MessageHistoryTableTests.swift */,
 			);
 			path = PostboxTests;
 			sourceTree = "<group>";
@@ -367,24 +388,29 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				D0F9E8631C579F0200037222 /* MediaCleanupTable.swift in Sources */,
 				D075165C1B2EC5B000AE42E0 /* module.private.modulemap in Sources */,
 				D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */,
 				D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */,
 				D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */,
 				D00E0FBE1B85D1B5002E4EB5 /* LmdbValueBox.swift in Sources */,
 				D07516721B2EC7FE00AE42E0 /* Value.swift in Sources */,
+				D08C713A1C501F0700779C0F /* MessageHistoryHole.swift in Sources */,
+				D0F9E85B1C565EBB00037222 /* MessageMediaTable.swift in Sources */,
+				D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */,
 				D0977FA41B829F87009994B2 /* PostboxTables.swift in Sources */,
 				D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */,
 				D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */,
 				D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */,
 				D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */,
-				D003E4E61B38DBDB00C22CBC /* MessageView.swift in Sources */,
+				D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */,
 				D075165E1B2EC5B500AE42E0 /* module.modulemap in Sources */,
 				D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */,
 				D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */,
 				D0D224F21B4D6ABD0085E26D /* Functions.swift in Sources */,
 				D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */,
 				D07516711B2EC7FE00AE42E0 /* Statement.swift in Sources */,
+				D08C713C1C51283C00779C0F /* MessageHistoryIndexTable.swift in Sources */,
 				D0D225261B4D84930085E26D /* PeerView.swift in Sources */,
 				D0E3A7841B28AE0900A402D9 /* Peer.swift in Sources */,
 				D055BD331B7D3D2D00F06C0A /* MediaBox.swift in Sources */,
@@ -401,6 +427,8 @@
 			buildActionMask = 2147483647;
 			files = (
 				D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */,
+				D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */,
+				D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/PostboxTests.xcscheme

@@ -10,7 +10,8 @@
       buildConfiguration = "Debug"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
-      shouldUseLaunchSchemeArgsEnv = "YES">
+      shouldUseLaunchSchemeArgsEnv = "YES"
+      codeCoverageEnabled = "YES">
       <Testables>
          <TestableReference
             skipped = "NO">

Postbox/Coding.swift

@@ -137,6 +137,10 @@ public final class ReadBuffer: MemoryBuffer {
         self.offset += length
     }
     
+    func skip(length: Int) {
+        self.offset += length
+    }
+    
     func reset() {
         self.offset = 0
     }

Postbox/LmdbValueBox.swift

@@ -98,6 +98,10 @@ public final class LmdbValueBox: ValueBox {
     
     private var sharedTxn: COpaquePointer = nil
     
+    private var readQueryTime: CFAbsoluteTime = 0.0
+    private var writeQueryTime: CFAbsoluteTime = 0.0
+    private var commitTime: CFAbsoluteTime = 0.0
+    
     public init?(basePath: String) {
         var result = mdb_env_create(&self.env)
         if result != MDB_SUCCESS {
@@ -163,6 +167,16 @@ public final class LmdbValueBox: ValueBox {
         return LmdbTable(dbi: dbi)
     }
     
+    public func beginStats() {
+        self.readQueryTime = 0.0
+        self.writeQueryTime = 0.0
+        self.commitTime = 0.0
+    }
+    
+    public func endStats() {
+        print("(LmdbValueBox stats read: \(self.readQueryTime * 1000.0) ms, write: \(self.writeQueryTime * 1000.0) ms, commit: \(self.commitTime * 1000.0) ms")
+    }
+    
     public func begin() {
         if self.sharedTxn != nil {
             print("(LmdbValueBox already in transaction)")
@@ -176,11 +190,13 @@ public final class LmdbValueBox: ValueBox {
     }
     
     public func commit() {
+        let startTime = CFAbsoluteTimeGetCurrent()
         if self.sharedTxn == nil {
             print("(LmdbValueBox already no current transaction)")
         } else {
             let result = mdb_txn_commit(self.sharedTxn)
             self.sharedTxn = nil
+            self.commitTime += CFAbsoluteTimeGetCurrent() - startTime
             if result != MDB_SUCCESS {
                 print("(LmdbValueBox txn_commit failed with \(result))")
                 return
@@ -208,6 +224,7 @@ public final class LmdbValueBox: ValueBox {
         }
         
         if let nativeTable = nativeTable {
+            var startTime = CFAbsoluteTimeGetCurrent()
             var cursorPtr: COpaquePointer = nil
             let result = mdb_cursor_open(self.sharedTxn, nativeTable.dbi, &cursorPtr)
             if result != MDB_SUCCESS {
@@ -223,6 +240,10 @@ public final class LmdbValueBox: ValueBox {
                         }
                     }
                     
+                    var currentTime = CFAbsoluteTimeGetCurrent()
+                    readQueryTime += currentTime - startTime
+                    startTime = currentTime
+                    
                     var count = 0
                     if value != nil && value!.0 < end {
                         count++
@@ -230,13 +251,21 @@ public final class LmdbValueBox: ValueBox {
                     }
                     
                     while value != nil && value!.0 < end && count < limit {
+                        startTime = CFAbsoluteTimeGetCurrent()
+                        
                         value = cursor.next()
+                        
+                        currentTime = CFAbsoluteTimeGetCurrent()
+                        readQueryTime += currentTime - startTime
+                        startTime = currentTime
+                        
                         if value != nil && value!.0 < end {
                             count++
                             values(value!.0, value!.1)
                         }
                     }
                 } else {
+                    var startTime = CFAbsoluteTimeGetCurrent()
                     var value = cursor.seekTo(start, forward: false)
                     if value != nil {
                         if value!.0 == start {
@@ -244,6 +273,10 @@ public final class LmdbValueBox: ValueBox {
                         }
                     }
                     
+                    var currentTime = CFAbsoluteTimeGetCurrent()
+                    readQueryTime += currentTime - startTime
+                    startTime = currentTime
+                    
                     var count = 0
                     if value != nil && value!.0 > end {
                         count++
@@ -251,7 +284,14 @@ public final class LmdbValueBox: ValueBox {
                     }
                     
                     while value != nil && value!.0 > end && count < limit {
+                        startTime = CFAbsoluteTimeGetCurrent()
+                        
                         value = cursor.previous()
+                        
+                        currentTime = CFAbsoluteTimeGetCurrent()
+                        readQueryTime += currentTime - startTime
+                        startTime = currentTime
+                        
                         if value != nil && value!.0 > end {
                             count++
                             values(value!.0, value!.1)
@@ -264,7 +304,11 @@ public final class LmdbValueBox: ValueBox {
         }
         
         if commit {
+            let startTime = CFAbsoluteTimeGetCurrent()
+            
             self.commit()
+            
+            readQueryTime += CFAbsoluteTimeGetCurrent() - startTime
         }
     }
     
@@ -275,6 +319,8 @@ public final class LmdbValueBox: ValueBox {
     }
     
     public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? {
+        let startTime = CFAbsoluteTimeGetCurrent()
+        
         var commit = false
         if self.sharedTxn == nil {
             self.begin()
@@ -315,6 +361,8 @@ public final class LmdbValueBox: ValueBox {
             self.commit()
         }
         
+        readQueryTime += CFAbsoluteTimeGetCurrent() - startTime
+        
         return resultValue
     }
     
@@ -323,6 +371,8 @@ public final class LmdbValueBox: ValueBox {
     }
     
     public func set(table: Int32, key: ValueBoxKey, value: MemoryBuffer) {
+        let startTime = CFAbsoluteTimeGetCurrent()
+        
         var commit = false
         if self.sharedTxn == nil {
             self.begin()
@@ -355,9 +405,13 @@ public final class LmdbValueBox: ValueBox {
         if commit {
             self.commit()
         }
+        
+        writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime
     }
     
     public func remove(table: Int32, key: ValueBoxKey) {
+        let startTime = CFAbsoluteTimeGetCurrent()
+        
         var commit = false
         if self.sharedTxn == nil {
             self.begin()
@@ -387,6 +441,8 @@ public final class LmdbValueBox: ValueBox {
         if commit {
             self.commit()
         }
+        
+        writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime
     }
     
     public func drop() {

Postbox/Media.swift

@@ -25,11 +25,13 @@ public struct MediaId: Hashable, CustomStringConvertible {
     }
     
     public init(_ buffer: ReadBuffer) {
-        self.namespace = 0
+        var namespace: Int32 = 0
-        self.id = 0
+        var id: Int64 = 0
         
-        memcpy(&self.namespace, buffer.memory + buffer.offset, 4)
+        memcpy(&namespace, buffer.memory + buffer.offset, 4)
-        memcpy(&self.id, buffer.memory + (buffer.offset + 4), 8)
+        self.namespace = namespace
+        memcpy(&id, buffer.memory + (buffer.offset + 4), 8)
+        self.id = id
         buffer.offset += 12
     }
     
@@ -68,4 +70,6 @@ public func ==(lhs: MediaId, rhs: MediaId) -> Bool {
 
 public protocol Media: Coding {
     var id: MediaId? { get }
+    
+    func isEqual(other: Media) -> Bool
 }

Postbox/MediaCleanupTable.swift

@@ -0,0 +1,21 @@
+import Foundation
+
+final class MediaCleanupTable {
+    let valueBox: ValueBox
+    let tableId: Int32
+    
+    var debugMedia: [Media] = []
+    
+    init(valueBox: ValueBox, tableId: Int32) {
+        self.valueBox = valueBox
+        self.tableId = tableId
+    }
+    
+    func add(media: Media, sharedEncoder: Encoder = Encoder()) {
+        debugMedia.append(media)
+    }
+    
+    func debugList() -> [Media] {
+        return self.debugMedia
+    }
+}

Postbox/Message.swift

@@ -1,6 +1,6 @@
 import Foundation
 
-public struct MessageId: Hashable, CustomStringConvertible {
+public struct MessageId: Hashable, Comparable, CustomStringConvertible {
     public typealias Namespace = Int32
     public typealias Id = Int32
     
@@ -81,6 +81,14 @@ public func ==(lhs: MessageId, rhs: MessageId) -> Bool {
     return lhs.id == rhs.id && lhs.namespace == rhs.namespace
 }
 
+public func <(lhs: MessageId, rhs: MessageId) -> Bool {
+    if lhs.namespace == rhs.namespace {
+        return lhs.id < rhs.id
+    } else {
+        return lhs.namespace < rhs.namespace
+    }
+}
+
 public struct MessageIndex: Equatable, Comparable {
     public let id: MessageId
     public let timestamp: Int32
@@ -90,10 +98,23 @@ public struct MessageIndex: Equatable, Comparable {
         self.timestamp = message.timestamp
     }
     
+    init(_ message: StoreMessage) {
+        self.id = message.id
+        self.timestamp = message.timestamp
+    }
+    
     public init(id: MessageId, timestamp: Int32) {
         self.id = id
         self.timestamp = timestamp
     }
+    
+    public func predecessor() -> MessageIndex {
+        return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id - 1), timestamp: self.timestamp)
+    }
+    
+    public func successor() -> MessageIndex {
+        return MessageIndex(id: MessageId(peerId: self.id.peerId, namespace: self.id.namespace, id: self.id.id + 1), timestamp: self.timestamp)
+    }
 }
 
 public func ==(lhs: MessageIndex, rhs: MessageIndex) -> Bool {
@@ -112,12 +133,54 @@ public func <(lhs: MessageIndex, rhs: MessageIndex) -> Bool {
     return lhs.id.id < rhs.id.id
 }
 
-public protocol Message: Coding {
+public class Message {
-    var id: MessageId { get }
+    let id: MessageId
-    var timestamp: Int32 { get }
+    let timestamp: Int32
-    var text: String { get }
+    let text: String
-    var mediaIds: [MediaId] { get }
+    let attributes: [Coding]
-    var peerIds: [PeerId] { get }
+    let media: [Media]
+    
+    init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) {
+        self.id = id
+        self.timestamp = timestamp
+        self.text = text
+        self.attributes = attributes
+        self.media = media
+    }
+}
+
+class StoreMessage {
+    let id: MessageId
+    let timestamp: Int32
+    let text: String
+    let attributes: [Coding]
+    let media: [Media]
+    
+    init(id: MessageId, timestamp: Int32, text: String, attributes: [Coding], media: [Media]) {
+        self.id = id
+        self.timestamp = timestamp
+        self.text = text
+        self.attributes = attributes
+        self.media = media
+    }
+}
+
+class IntermediateMessage {
+    let id: MessageId
+    let timestamp: Int32
+    let text: String
+    let attributesData: ReadBuffer
+    let embeddedMediaData: ReadBuffer
+    let referencedMedia: [MediaId]
+    
+    init(id: MessageId, timestamp: Int32, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) {
+        self.id = id
+        self.timestamp = timestamp
+        self.text = text
+        self.attributesData = attributesData
+        self.embeddedMediaData = embeddedMediaData
+        self.referencedMedia = referencedMedia
+    }
 }
 
 public struct RenderedMessage: Equatable, Comparable {

Postbox/MessageHistoryHole.swift

@@ -0,0 +1,14 @@
+import Foundation
+
+public struct MessageHistoryHole: Equatable {
+    let maxIndex: MessageIndex
+    let min: MessageId.Id
+    
+    var id: MessageId {
+        return maxIndex.id
+    }
+}
+
+public func ==(lhs: MessageHistoryHole, rhs: MessageHistoryHole) -> Bool {
+    return lhs.maxIndex == rhs.maxIndex && lhs.min == rhs.min
+}

Postbox/MessageHistoryIndexTable.swift

@@ -0,0 +1,322 @@
+import Foundation
+
+enum HistoryIndexEntry {
+    case Message(MessageIndex)
+    case Hole(MessageHistoryHole)
+}
+
+enum HoleFillType {
+    case UpperToLower
+    case LowerToUpper
+    case Complete
+}
+
+private func readHistoryIndexEntry(peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> HistoryIndexEntry {
+    var type: Int8 = 0
+    value.read(&type, offset: 0, length: 1)
+    var timestamp: Int32 = 0
+    value.read(&timestamp, offset: 0, length: 4)
+    let index = MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: key.getInt32(8 + 4)), timestamp: timestamp)
+    
+    if type == 0 {
+        return .Message(index)
+    } else {
+        var min: Int32 = 0
+        value.read(&min, offset: 0, length: 4)
+        return .Hole(MessageHistoryHole(maxIndex: index, min: min))
+    }
+}
+
+final class MessageHistoryIndexTable {
+    let valueBox: ValueBox
+    let tableId: Int32
+    
+    init(valueBox: ValueBox, tableId: Int32) {
+        self.valueBox = valueBox
+        self.tableId = tableId
+    }
+    
+    func key(id: MessageId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4 + 4)
+        key.setInt64(0, value: id.peerId.toInt64())
+        key.setInt32(8, value: id.namespace)
+        key.setInt32(8 + 4, value: id.id)
+        return key
+    }
+    
+    func lowerBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        return key
+    }
+    
+    func upperBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        return key.successor
+    }
+    
+    func addHole(id: MessageId) {
+        let adjacent = self.adjacentItems(id)
+        
+        /*
+        
+        1. [x] nothing
+        2. [x] message - * - nothing
+        3. [x] nothing - * - message
+        4. [x] nothing - * - hole
+        5. [x] message - * - message
+        6. [x] hole - * - message
+        7. [x] message - * - hole
+        
+        */
+        
+        if let lowerItem = adjacent.lower, upperItem = adjacent.upper {
+            switch lowerItem {
+                case .Hole:
+                    break
+                case let .Message(lowerMessage):
+                    switch upperItem {
+                        case .Hole:
+                            break
+                        case let .Message(upperMessage):
+                            if lowerMessage.id.id < upperMessage.id.id - 1 {
+                                self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerMessage.id.id + 1))
+                            }
+                            break
+                    }
+            }
+        } else if let lowerItem = adjacent.lower {
+            switch lowerItem {
+                case let .Message(lowerMessage):
+                    self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerMessage.id.id + 1))
+                case .Hole:
+                    break
+            }
+        } else if let upperItem = adjacent.upper {
+            switch upperItem {
+                case let .Message(upperMessage):
+                    if upperMessage.id.id > 1 {
+                        self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: 1))
+                    }
+                case .Hole:
+                    break
+            }
+        } else {
+            self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: 1))
+        }
+    }
+    
+    func addMessage(index: MessageIndex) {
+        var upperItem: HistoryIndexEntry?
+        self.valueBox.range(self.tableId, start: self.key(index.id).predecessor, end: self.upperBound(index.id.peerId, namespace: index.id.namespace), values: { key, value in
+            upperItem = readHistoryIndexEntry(index.id.peerId, namespace: index.id.namespace, key: key, value: value)
+            return true
+        }, limit: 1)
+        
+        if let upperItem = upperItem {
+            switch upperItem {
+                case let .Hole(upperHole):
+                    self.justRemove(upperHole.id)
+                    if upperHole.maxIndex.id.id > index.id.id + 1 {
+                        self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: index.id.id + 1))
+                    }
+                    if upperHole.min <= index.id.id - 1 {
+                        self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: index.id.peerId, namespace: index.id.namespace, id: index.id.id - 1), timestamp: index.timestamp), min: upperHole.min))
+                    }
+                    break
+                case .Message:
+                    break
+            }
+        }
+        
+        self.justInsertMessage(index)
+    }
+    
+    func removeMessage(id: MessageId) {
+        let key = self.key(id)
+        if self.valueBox.exists(self.tableId, key: key) {
+            self.justRemove(id)
+            
+            let adjacent = self.adjacentItems(id)
+            
+            if let lowerItem = adjacent.lower, upperItem = adjacent.upper {
+                switch lowerItem {
+                    case let .Message(lowerMessage):
+                        switch upperItem {
+                            case let .Hole(upperHole):
+                                self.justRemove(upperHole.id)
+                                self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerMessage.id.id + 1))
+                            case .Message:
+                                break
+                        }
+                    case let .Hole(lowerHole):
+                        switch upperItem {
+                            case let .Hole(upperHole):
+                                self.justRemove(lowerHole.id)
+                                self.justRemove(upperHole.id)
+                                self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: lowerHole.min))
+                            case let .Message(upperMessage):
+                                self.justRemove(lowerHole.id)
+                                self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: upperMessage.id.id - 1), timestamp: upperMessage.timestamp), min: lowerHole.min))
+                        }
+                }
+            } else if let lowerItem = adjacent.lower {
+                switch lowerItem {
+                    case let .Hole(lowerHole):
+                        self.justRemove(lowerHole.id)
+                        self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: Int32.max), timestamp: Int32.max), min: lowerHole.min))
+                        break
+                    case .Message:
+                        break
+                }
+            } else if let upperItem = adjacent.upper {
+                switch upperItem {
+                    case let .Hole(upperHole):
+                        self.justRemove(upperHole.id)
+                        self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: 1))
+                        break
+                    case .Message:
+                        break
+                }
+            }
+        }
+    }
+    
+    func fillHole(id: MessageId, fillType: HoleFillType, indices: [MessageIndex]) {
+        var upperItem: HistoryIndexEntry?
+        self.valueBox.range(self.tableId, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in
+            upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+            return true
+        }, limit: 1)
+        
+        let sortedByIdIndices = indices.sort({$0.id < $1.id})
+        var remainingIndices = sortedByIdIndices
+        
+        if let upperItem = upperItem {
+            switch upperItem {
+                case let .Hole(upperHole):
+                    var i = 0
+                    var minIndexInRange: MessageIndex?
+                    var maxIndexInRange: MessageIndex?
+                    var removedHole = false
+                    while i < remainingIndices.count {
+                        let index = remainingIndices[i]
+                        if index.id.id >= upperHole.min && index.id.id <= upperHole.maxIndex.id.id {
+                            if (fillType == .UpperToLower || fillType == .Complete) && (minIndexInRange == nil || minIndexInRange!.id > index.id) {
+                                minIndexInRange = index
+                                if !removedHole {
+                                    removedHole = true
+                                    self.justRemove(upperHole.id)
+                                }
+                            }
+                            if (fillType == .LowerToUpper || fillType == .Complete) && (maxIndexInRange == nil || maxIndexInRange!.id < index.id) {
+                                maxIndexInRange = index
+                                if !removedHole {
+                                    removedHole = true
+                                    self.justRemove(upperHole.id)
+                                }
+                            }
+                            self.justInsertMessage(index)
+                            remainingIndices.removeAtIndex(i)
+                        } else {
+                            i++
+                        }
+                    }
+                    switch fillType {
+                        case .Complete:
+                            if !removedHole {
+                                self.justRemove(upperHole.id)
+                            }
+                        case .LowerToUpper:
+                            if let maxIndexInRange = maxIndexInRange where maxIndexInRange.id.id != Int32.max && maxIndexInRange.id.id + 1 <= upperHole.maxIndex.id.id {
+                                self.justInsertHole(MessageHistoryHole(maxIndex: upperHole.maxIndex, min: maxIndexInRange.id.id + 1))
+                            }
+                        case .UpperToLower:
+                            if let minIndexInRange = minIndexInRange where minIndexInRange.id.id - 1 >= upperHole.min {
+                                self.justInsertHole(MessageHistoryHole(maxIndex: MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: minIndexInRange.id.id - 1), timestamp: minIndexInRange.timestamp), min: upperHole.min))
+                            }
+                    }
+                    break
+                case .Message:
+                    break
+            }
+        }
+        
+        for index in remainingIndices {
+            self.addMessage(index)
+        }
+    }
+    
+    func justInsertHole(hole: MessageHistoryHole) {
+        let value = WriteBuffer()
+        var type: Int8 = 1
+        var timestamp: Int32 = hole.maxIndex.timestamp
+        var min: Int32 = hole.min
+        value.write(&type, offset: 0, length: 1)
+        value.write(&timestamp, offset: 0, length: 4)
+        value.write(&min, offset: 0, length: 4)
+        self.valueBox.set(self.tableId, key: self.key(hole.id), value: value)
+    }
+    
+    func justInsertMessage(index: MessageIndex) {
+        let value = WriteBuffer()
+        var type: Int8 = 0
+        var timestamp: Int32 = index.timestamp
+        value.write(&type, offset: 0, length: 1)
+        value.write(&timestamp, offset: 0, length: 4)
+        self.valueBox.set(self.tableId, key: self.key(index.id), value: value)
+    }
+    
+    func justRemove(id: MessageId) {
+        self.valueBox.remove(self.tableId, key: self.key(id))
+    }
+    
+    func adjacentItems(id: MessageId) -> (lower: HistoryIndexEntry?, upper: HistoryIndexEntry?) {
+        let key = self.key(id)
+        
+        var lowerItem: HistoryIndexEntry?
+        self.valueBox.range(self.tableId, start: key, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in
+            lowerItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+            return true
+        }, limit: 1)
+        
+        var upperItem: HistoryIndexEntry?
+        self.valueBox.range(self.tableId, start: key.predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in
+            upperItem = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+            return true
+        }, limit: 1)
+        
+        return (lower: lowerItem, upper: upperItem)
+    }
+    
+    func get(id: MessageId) -> HistoryIndexEntry? {
+        let key = self.key(id)
+        if let value = self.valueBox.get(self.tableId, key: key) {
+            return readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+        }
+        return nil
+    }
+    
+    func messageExists(id: MessageId) -> Bool {
+        if let entry = self.get(id) {
+            if case .Message = entry {
+                return true
+            }
+        }
+        
+        return false
+    }
+    
+    func debugList(peerId: PeerId, namespace: MessageId.Namespace) -> [HistoryIndexEntry] {
+        var list: [HistoryIndexEntry] = []
+        self.valueBox.range(self.tableId, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in
+            list.append(readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value))
+            
+            return true
+        }, limit: 0)
+        return list
+    }
+}

Postbox/MessageHistoryTable.swift

@@ -0,0 +1,359 @@
+import Foundation
+
+enum MessageHistoryEntry {
+    case Msg(Message)
+    case Hole(MessageHistoryHole)
+    
+    var index: MessageIndex {
+        switch self {
+            case let .Msg(message):
+                return MessageIndex(message)
+            case let .Hole(hole):
+                return hole.maxIndex
+        }
+    }
+}
+
+final class MessageHistoryTable {
+    let valueBox: ValueBox
+    let tableId: Int32
+    
+    let messageHistoryIndexTable: MessageHistoryIndexTable
+    let messageMediaTable: MessageMediaTable
+    
+    init(valueBox: ValueBox, tableId: Int32, messageHistoryIndexTable: MessageHistoryIndexTable, messageMediaTable: MessageMediaTable) {
+        self.valueBox = valueBox
+        self.tableId = tableId
+        self.messageHistoryIndexTable = messageHistoryIndexTable
+        self.messageMediaTable = messageMediaTable
+    }
+    
+    private func key(index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4)) -> ValueBoxKey {
+        key.setInt64(0, value: index.id.peerId.toInt64())
+        key.setInt32(8, value: index.timestamp)
+        key.setInt32(8 + 4, value: index.id.namespace)
+        key.setInt32(8 + 4 + 4, value: index.id.id)
+        return key
+    }
+    
+    private func lowerBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key.successor
+    }
+    
+    private func messagesByPeerId(messages: [StoreMessage]) -> [PeerId: [StoreMessage]] {
+        var dict: [PeerId: [StoreMessage]] = [:]
+        
+        for message in messages {
+            let peerId = message.id.peerId
+            if dict[peerId] == nil {
+                dict[peerId] = [message]
+            } else {
+                dict[peerId]!.append(message)
+            }
+        }
+        
+        return dict
+    }
+    
+    func addMessages(messages: [StoreMessage]) {
+        let sharedKey = self.key(MessageIndex(id: MessageId(peerId: PeerId(namespace: 0, id: 0), namespace: 0, id: 0), timestamp: 0))
+        let sharedBuffer = WriteBuffer()
+        let sharedEncoder = Encoder()
+        
+        let messagesByPeerId = self.messagesByPeerId(messages)
+        for (_, peerMessages) in messagesByPeerId {
+            for message in peerMessages {
+                if self.messageHistoryIndexTable.messageExists(message.id) {
+                    continue
+                }
+                
+                self.messageHistoryIndexTable.addMessage(MessageIndex(message))
+                self.justInsert(message, sharedKey: sharedKey, sharedBuffer: sharedBuffer, sharedEncoder: sharedEncoder)
+            }
+        }
+    }
+    
+    func removeMessages(messageIds: [MessageId]) {
+        for messageId in messageIds {
+            if let entry = self.messageHistoryIndexTable.get(messageId) {
+                if case let .Message(index) = entry {
+                    if let message = self.get(index) {
+                        let embeddedMediaData = message.embeddedMediaData
+                        if embeddedMediaData.length > 4 {
+                            var embeddedMediaCount: Int32 = 0
+                            embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4)
+                            for _ in 0 ..< embeddedMediaCount {
+                                var mediaLength: Int32 = 0
+                                embeddedMediaData.read(&mediaLength, offset: 0, length: 4)
+                                if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                                    self.messageMediaTable.removeEmbeddedMedia(media)
+                                }
+                                embeddedMediaData.skip(Int(mediaLength))
+                            }
+                        }
+                        
+                        for mediaId in message.referencedMedia {
+                            self.messageMediaTable.removeReference(mediaId)
+                        }
+                    }
+                    
+                    self.messageHistoryIndexTable.removeMessage(messageId)
+                    self.valueBox.remove(self.tableId, key: self.key(index))
+                }
+            }
+        }
+    }
+    
+    private func justInsert(message: StoreMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer, sharedEncoder: Encoder) {
+        sharedBuffer.reset()
+
+        let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!
+        var length: Int32 = Int32(data.length)
+        sharedBuffer.write(&length, offset: 0, length: 4)
+        sharedBuffer.write(data.bytes, offset: 0, length: Int(length))
+
+        var attributeCount: Int32 = Int32(message.attributes.count)
+        sharedBuffer.write(&attributeCount, offset: 0, length: 4)
+        for attribute in message.attributes {
+            sharedEncoder.reset()
+            sharedEncoder.encodeRootObject(attribute)
+            let attributeBuffer = sharedEncoder.memoryBuffer()
+            var attributeBufferLength = Int32(attributeBuffer.length)
+            sharedBuffer.write(&attributeBufferLength, offset: 0, length: 4)
+            sharedBuffer.write(attributeBuffer.memory, offset: 0, length: attributeBuffer.length)
+        }
+        
+        var embeddedMedia: [Media] = []
+        var referencedMedia: [MediaId] = []
+        for media in message.media {
+            if let mediaId = media.id {
+                let mediaInsertResult = self.messageMediaTable.set(media, index: MessageIndex(message), messageHistoryTable: self)
+                switch mediaInsertResult {
+                    case let .Embed(media):
+                        embeddedMedia.append(media)
+                    case .Reference:
+                        referencedMedia.append(mediaId)
+                }
+            } else {
+                embeddedMedia.append(media)
+            }
+        }
+        
+        var embeddedMediaCount: Int32 = Int32(embeddedMedia.count)
+        sharedBuffer.write(&embeddedMediaCount, offset: 0, length: 4)
+        for media in embeddedMedia {
+            sharedEncoder.reset()
+            sharedEncoder.encodeRootObject(media)
+            let mediaBuffer = sharedEncoder.memoryBuffer()
+            var mediaBufferLength = Int32(mediaBuffer.length)
+            sharedBuffer.write(&mediaBufferLength, offset: 0, length: 4)
+            sharedBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length)
+        }
+        
+        var referencedMediaCount: Int32 = Int32(referencedMedia.count)
+        sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4)
+        for mediaId in referencedMedia {
+            var idNamespace: Int32 = mediaId.namespace
+            var idId: Int64 = mediaId.id
+            sharedBuffer.write(&idNamespace, offset: 0, length: 4)
+            sharedBuffer.write(&idId, offset: 0, length: 8)
+        }
+        
+        self.valueBox.set(self.tableId, key: self.key(MessageIndex(message), key: sharedKey), value: sharedBuffer)
+    }
+    
+    func unembedMedia(index: MessageIndex, id: MediaId) -> Media? {
+        if let message = self.get(index) where message.embeddedMediaData.length > 4 {
+            var embeddedMediaCount: Int32 = 0
+            message.embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4)
+            
+            let updatedEmbeddedMediaBuffer = WriteBuffer()
+            var updatedEmbeddedMediaCount = embeddedMediaCount - 1
+            updatedEmbeddedMediaBuffer.write(&updatedEmbeddedMediaCount, offset: 0, length: 4)
+            
+            var extractedMedia: Media?
+            
+            for _ in 0 ..< embeddedMediaCount {
+                let mediaOffset = message.embeddedMediaData.offset
+                var mediaLength: Int32 = 0
+                var copyMedia = true
+                message.embeddedMediaData.read(&mediaLength, offset: 0, length: 4)
+                if let media = Decoder(buffer: MemoryBuffer(memory: message.embeddedMediaData.memory + message.embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                    
+                    if let mediaId = media.id where mediaId == id {
+                        copyMedia = false
+                        extractedMedia = media
+                    }
+                }
+                
+                if copyMedia {
+                    updatedEmbeddedMediaBuffer.write(message.embeddedMediaData.memory + mediaOffset, offset: 0, length: message.embeddedMediaData.offset - mediaOffset)
+                }
+            }
+            
+            if let extractedMedia = extractedMedia {
+                var updatedReferencedMedia = message.referencedMedia
+                updatedReferencedMedia.append(extractedMedia.id!)
+                self.storeIntermediateMessage(IntermediateMessage(id: message.id, timestamp: message.timestamp, text: message.text, attributesData: message.attributesData, embeddedMediaData: updatedEmbeddedMediaBuffer.readBufferNoCopy(), referencedMedia: updatedReferencedMedia), sharedKey: self.key(index))
+                
+                return extractedMedia
+            }
+        }
+        return nil
+    }
+    
+    func storeIntermediateMessage(message: IntermediateMessage, sharedKey: ValueBoxKey, sharedBuffer: WriteBuffer = WriteBuffer()) {
+        sharedBuffer.reset()
+        
+        let data = message.text.dataUsingEncoding(NSUTF8StringEncoding, allowLossyConversion: true)!
+        var length: Int32 = Int32(data.length)
+        sharedBuffer.write(&length, offset: 0, length: 4)
+        sharedBuffer.write(data.bytes, offset: 0, length: Int(length))
+        
+        sharedBuffer.write(message.attributesData.memory, offset: 0, length: message.attributesData.length)
+        sharedBuffer.write(message.embeddedMediaData.memory, offset: 0, length: message.embeddedMediaData.length)
+        
+        var referencedMediaCount: Int32 = Int32(message.referencedMedia.count)
+        sharedBuffer.write(&referencedMediaCount, offset: 0, length: 4)
+        for mediaId in message.referencedMedia {
+            var idNamespace: Int32 = mediaId.namespace
+            var idId: Int64 = mediaId.id
+            sharedBuffer.write(&idNamespace, offset: 0, length: 4)
+            sharedBuffer.write(&idId, offset: 0, length: 8)
+        }
+        
+        self.valueBox.set(self.tableId, key: self.key(MessageIndex(id: message.id, timestamp: message.timestamp), key: sharedKey), value: sharedBuffer)
+    }
+    
+    private func readIntermediateMessage(key: ValueBoxKey, value: ReadBuffer) -> IntermediateMessage {
+        let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4)), timestamp: key.getInt32(8))
+        
+        var textLength: Int32 = 0
+        value.read(&textLength, offset: 0, length: 4)
+        let text = String(data: NSData(bytes: value.memory + value.offset, length: Int(textLength)), encoding: NSUTF8StringEncoding) ?? ""
+        value.skip(Int(textLength))
+        
+        let attributesOffset = value.offset
+        var attributeCount: Int32 = 0
+        value.read(&attributeCount, offset: 0, length: 4)
+        for _ in 0 ..< attributeCount {
+            var attributeLength: Int32 = 0
+            value.read(&attributeLength, offset: 0, length: 4)
+            value.skip(Int(attributeLength))
+        }
+        let attributesLength = value.offset - attributesOffset
+        let attributesBytes = malloc(attributesLength)
+        memcpy(attributesBytes, value.memory + attributesOffset, attributesLength)
+        let attributesData = ReadBuffer(memory: attributesBytes, length: attributesLength, freeWhenDone: true)
+        
+        let embeddedMediaOffset = value.offset
+        var embeddedMediaCount: Int32 = 0
+        value.read(&embeddedMediaCount, offset: 0, length: 4)
+        for _ in 0 ..< embeddedMediaCount {
+            var mediaLength: Int32 = 0
+            value.read(&mediaLength, offset: 0, length: 4)
+            value.skip(Int(mediaLength))
+        }
+        let embeddedMediaLength = value.offset - embeddedMediaOffset
+        let embeddedMediaBytes = malloc(embeddedMediaLength)
+        memcpy(embeddedMediaBytes, value.memory + embeddedMediaOffset, embeddedMediaLength)
+        let embeddedMediaData = ReadBuffer(memory: embeddedMediaBytes, length: embeddedMediaLength, freeWhenDone: true)
+        
+        var referencedMediaIds: [MediaId] = []
+        var referencedMediaIdsCount: Int32 = 0
+        value.read(&referencedMediaIdsCount, offset: 0, length: 4)
+        for _ in 0 ..< referencedMediaIdsCount {
+            var idNamespace: Int32 = 0
+            var idId: Int64 = 0
+            value.read(&idNamespace, offset: 0, length: 4)
+            value.read(&idId, offset: 0, length: 8)
+            referencedMediaIds.append(MediaId(namespace: idNamespace, id: idId))
+        }
+        
+        return IntermediateMessage(id: index.id, timestamp: index.timestamp, text: text, attributesData: attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: referencedMediaIds)
+    }
+    
+    func get(index: MessageIndex) -> IntermediateMessage? {
+        let key = self.key(index)
+        if let value = self.valueBox.get(self.tableId, key: key) {
+            return self.readIntermediateMessage(key, value: value)
+        }
+        return nil
+    }
+    
+    func renderMessage(message: IntermediateMessage) -> Message {
+        var parsedAttributes: [Coding] = []
+        var parsedMedia: [Media] = []
+        
+        let attributesData = message.attributesData
+        if attributesData.length > 4 {
+            var attributeCount: Int32 = 0
+            attributesData.read(&attributeCount, offset: 0, length: 4)
+            for _ in 0 ..< attributeCount {
+                var attributeLength: Int32 = 0
+                attributesData.read(&attributeLength, offset: 0, length: 4)
+                if let attribute = Decoder(buffer: MemoryBuffer(memory: attributesData.memory + attributesData.offset, capacity: Int(attributeLength), length: Int(attributeLength), freeWhenDone: false)).decodeRootObject() {
+                    parsedAttributes.append(attribute)
+                }
+                attributesData.skip(Int(attributeLength))
+            }
+        }
+        
+        let embeddedMediaData = message.embeddedMediaData
+        if embeddedMediaData.length > 4 {
+            var embeddedMediaCount: Int32 = 0
+            embeddedMediaData.read(&embeddedMediaCount, offset: 0, length: 4)
+            for _ in 0 ..< embeddedMediaCount {
+                var mediaLength: Int32 = 0
+                embeddedMediaData.read(&mediaLength, offset: 0, length: 4)
+                if let media = Decoder(buffer: MemoryBuffer(memory: embeddedMediaData.memory + embeddedMediaData.offset, capacity: Int(mediaLength), length: Int(mediaLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                    parsedMedia.append(media)
+                }
+                embeddedMediaData.skip(Int(mediaLength))
+            }
+        }
+        
+        for mediaId in message.referencedMedia {
+            if let media = self.messageMediaTable.get(mediaId) {
+                parsedMedia.append(media)
+            }
+        }
+        
+        return Message(id: message.id, timestamp: message.timestamp, text: message.text, attributes: parsedAttributes, media: parsedMedia)
+    }
+    
+    func messagesAround(index: MessageIndex, count: Int) -> [IntermediateMessage] {
+        var lowerMessages: [IntermediateMessage] = []
+        var upperMessages: [IntermediateMessage] = []
+        
+        self.valueBox.range(self.tableId, start: self.key(index), end: self.lowerBound(index.id.peerId), values: { key, value in
+            lowerMessages.append(self.readIntermediateMessage(key, value: value))
+            return true
+        }, limit: count)
+        
+        self.valueBox.range(self.tableId, start: self.key(index).predecessor, end: self.upperBound(index.id.peerId), values: { key, value in
+            upperMessages.append(self.readIntermediateMessage(key, value: value))
+            return true
+        }, limit: count)
+        
+        var messages: [IntermediateMessage] = []
+        for message in lowerMessages.reverse() {
+            messages.append(message)
+        }
+        messages.appendContentsOf(upperMessages)
+        
+        return messages
+    }
+    
+    func debugList(peerId: PeerId) -> [Message] {
+        return self.messagesAround(MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0), count: 1000).map({self.renderMessage($0)})
+    }
+}

Postbox/MessageHistoryView.swift

@@ -0,0 +1,341 @@
+import Foundation
+
+public final class MutableMessageHistoryView: CustomStringConvertible {
+    public struct RemoveContext {
+        var invalidEarlier: Bool = false
+        var invalidLater: Bool = false
+        var invalidEarlierHole: Bool = false
+        var invalidLaterHole: Bool = false
+        var removedMessages: Bool = false
+        var removedHole: Bool = false
+        
+        func empty() -> Bool {
+            return !self.removedMessages && !invalidEarlier && !invalidLater
+        }
+    }
+    
+    let count: Int
+    var earlierMessage: RenderedMessage?
+    var laterMessage: RenderedMessage?
+    var messages: [RenderedMessage]
+    
+    public init(count: Int, earlierMessage: RenderedMessage?, messages: [RenderedMessage], laterMessage: RenderedMessage?) {
+        self.count = count
+        self.earlierMessage = earlierMessage
+        self.laterMessage = laterMessage
+        self.messages = messages
+    }
+    
+    public func add(message: RenderedMessage) -> Bool {
+        if self.messages.count == 0 {
+            self.messages.append(message)
+            return true
+        } else {
+            let first = MessageIndex(self.messages[self.messages.count - 1].message)
+            let last = MessageIndex(self.messages[0].message)
+            
+            var next: MessageIndex?
+            if let message = laterMessage {
+                let messageIndex = MessageIndex(message.message)
+                next = messageIndex
+            }
+            
+            let index = MessageIndex(message.message)
+            
+            if index < last {
+                let earlierMessage = self.earlierMessage
+                if earlierMessage == nil || MessageIndex(earlierMessage!.message) < index {
+                    if self.messages.count < self.count {
+                        self.messages.insert(message, atIndex: 0)
+                    } else {
+                        self.earlierMessage = message
+                    }
+                    return true
+                } else {
+                    return false
+                }
+            } else if index > first {
+                if next != nil && index > next! {
+                    let laterMessage = self.laterMessage
+                    if laterMessage == nil || MessageIndex(laterMessage!.message) > index {
+                        if self.messages.count < self.count {
+                            self.messages.append(message)
+                        } else {
+                            self.laterMessage = message
+                        }
+                        return true
+                    } else {
+                        return false
+                    }
+                } else {
+                    self.messages.append(message)
+                    if self.messages.count > self.count {
+                        let earliest = self.messages[0]
+                        self.earlierMessage = earliest
+                        self.messages.removeAtIndex(0)
+                    }
+                    return true
+                }
+            } else if index != last && index != first {
+                var i = self.messages.count
+                while i >= 1 {
+                    if MessageIndex(self.messages[i - 1].message) < index {
+                        break
+                    }
+                    i--
+                }
+                self.messages.insert(message, atIndex: i)
+                if self.messages.count > self.count {
+                    let earliest = self.messages[0]
+                    self.earlierMessage = earliest
+                    self.messages.removeAtIndex(0)
+                }
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    public func remove(ids: Set<MessageId>, context: RemoveContext? = nil) -> RemoveContext {
+        var updatedContext = RemoveContext()
+        if let context = context {
+            updatedContext = context
+        }
+        
+        if let earlierMessage = self.earlierMessage where ids.contains(earlierMessage.message.id) {
+            updatedContext.invalidEarlier = true
+        }
+        
+        if let laterMessage = self.laterMessage where ids.contains(laterMessage.message.id) {
+            updatedContext.invalidLater = true
+        }
+        
+        if self.messages.count != 0 {
+            var i = self.messages.count - 1
+            while i >= 0 {
+                if ids.contains(self.messages[i].message.id) {
+                    self.messages.removeAtIndex(i)
+                    updatedContext.removedMessages = true
+                }
+                i--
+            }
+        }
+        
+        return updatedContext
+    }
+    
+    public func complete(context: RemoveContext, fetchEarlier: (MessageIndex?, Int) -> [RenderedMessage], fetchLater: (MessageIndex?, Int) -> [RenderedMessage]) {
+        if context.removedMessages {
+            var addedMessages: [RenderedMessage] = []
+            
+            var latestAnchor: MessageIndex?
+            if let lastMessage = self.messages.last {
+                latestAnchor = MessageIndex(lastMessage.message)
+            }
+            
+            if latestAnchor == nil {
+                if let laterMessage = self.laterMessage {
+                    latestAnchor = MessageIndex(laterMessage.message)
+                }
+            }
+            
+            if let laterMessage = self.laterMessage {
+                addedMessages += fetchLater(MessageIndex(laterMessage.message).predecessor(), self.count)
+            }
+            if let earlierMessage = self.earlierMessage {
+                addedMessages += fetchEarlier(MessageIndex(earlierMessage.message).successor(), self.count)
+            }
+            
+            addedMessages += self.messages
+            addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) })
+            var i = addedMessages.count - 1
+            while i >= 1 {
+                if addedMessages[i].message.id == addedMessages[i - 1].message.id {
+                    addedMessages.removeAtIndex(i)
+                }
+                i--
+            }
+            self.messages = []
+            
+            var anchorIndex = addedMessages.count - 1
+            if let latestAnchor = latestAnchor {
+                var i = addedMessages.count - 1
+                while i >= 0 {
+                    if MessageIndex(addedMessages[i].message) <= latestAnchor {
+                        anchorIndex = i
+                        break
+                    }
+                    i--
+                }
+            }
+            
+            self.laterMessage = nil
+            if anchorIndex + 1 < addedMessages.count {
+                self.laterMessage = addedMessages[anchorIndex + 1]
+            }
+            
+            i = anchorIndex
+            while i >= 0 && i > anchorIndex - self.count {
+                self.messages.insert(addedMessages[i], atIndex: 0)
+                i--
+            }
+            
+            self.earlierMessage = nil
+            if anchorIndex - self.count >= 0 {
+                self.earlierMessage = addedMessages[anchorIndex - self.count]
+            }
+        }
+        else {
+            if context.invalidEarlier {
+                var earlyId: MessageIndex?
+                let i = 0
+                if i < self.messages.count {
+                    earlyId = MessageIndex(self.messages[i].message)
+                }
+                
+                let earlierMessages = fetchEarlier(earlyId, 1)
+                self.earlierMessage = earlierMessages.first
+            }
+            
+            if context.invalidLater {
+                var laterId: MessageIndex?
+                let i = self.messages.count - 1
+                if i >= 0 {
+                    laterId = MessageIndex(self.messages[i].message)
+                }
+                
+                let laterMessages = fetchLater(laterId, 1)
+                self.laterMessage = laterMessages.first
+            }
+        }
+    }
+    
+    
+    public func incompleteMessages() -> [Message] {
+        var result: [Message] = []
+        
+        if let earlierMessage = self.earlierMessage where earlierMessage.incomplete {
+            result.append(earlierMessage.message)
+        }
+
+        if let laterMessage = self.laterMessage where laterMessage.incomplete {
+            result.append(laterMessage.message)
+        }
+        
+        for message in self.messages {
+            if message.incomplete {
+                result.append(message.message)
+            }
+        }
+        
+        return result
+    }
+    
+    public func completeMessages(messages: [MessageId : RenderedMessage]) {
+        if let earlierMessage = self.earlierMessage {
+            if let renderedMessage = messages[earlierMessage.message.id] {
+                self.earlierMessage = renderedMessage
+            }
+        }
+        
+        if let laterMessage = self.laterMessage {
+            if let renderedMessage = messages[laterMessage.message.id] {
+                self.laterMessage = renderedMessage
+            }
+        }
+        
+        var i = 0
+        while i < self.messages.count {
+            if let message = messages[self.messages[i].message.id] {
+                self.messages[i] = message
+            }
+            i++
+        }
+    }
+    
+    public var description: String {
+        var string = ""
+        string += "...("
+        if let value = self.earlierMessage {
+            string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)"
+        }
+        string += ") —— "
+        
+        string += "["
+        var first = true
+        for message in self.messages {
+            if first {
+                first = false
+            } else {
+                string += ", "
+            }
+            string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)"
+        }
+        string += "]"
+        
+        string += " —— ("
+        if let value = self.laterMessage {
+            string += "\(value.message.id.namespace): \(value.message.id.id)—\(value.message.timestamp)"
+        }
+        string += ")..."
+        
+        return string
+    }
+}
+
+public final class MessageHistoryView: CustomStringConvertible {
+    public let hasEarlier: Bool
+    private let earlierId: MessageIndex?
+    public let hasLater: Bool
+    private let laterId: MessageIndex?
+    public let messages: [RenderedMessage]
+    
+    init(_ mutableView: MutableMessageHistoryView) {
+        self.hasEarlier = mutableView.earlierMessage != nil
+        self.hasLater = mutableView.laterMessage != nil
+        self.messages = mutableView.messages
+        
+        if let earlierMessage = mutableView.earlierMessage {
+            self.earlierId = MessageIndex(earlierMessage.message)
+        } else {
+            self.earlierId = nil
+        }
+        
+        if let laterMessage = mutableView.laterMessage {
+            self.laterId = MessageIndex(laterMessage.message)
+        } else {
+            self.laterId = nil
+        }
+    }
+    
+    public var description: String {
+        var string = ""
+        if self.hasEarlier {
+            string += "more("
+            if let earlierId = self.earlierId {
+                string += "\(earlierId.id.namespace): \(earlierId.id.id)—\(earlierId.timestamp)"
+            }
+            string += ") "
+        }
+        string += "["
+        var first = true
+        for message in self.messages {
+            if first {
+                first = false
+            } else {
+                string += ", "
+            }
+            string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)"
+        }
+        string += "]"
+        if self.hasLater {
+            string += " more("
+            if let laterId = self.laterId {
+                string += "\(laterId.id.namespace): \(laterId.id.id)—\(laterId.timestamp)"
+            }
+            string += ")"
+        }
+        return string
+    }
+}

Postbox/MessageMediaTable.swift

@@ -0,0 +1,210 @@
+import Foundation
+
+private enum MediaEntryType: Int8 {
+    case Direct
+    case MessageReference
+}
+
+enum InsertMediaResult {
+    case Reference
+    case Embed(Media)
+}
+
+enum DebugMediaEntry {
+    case Direct(Media, Int)
+    case MessageReference(MessageIndex)
+}
+
+final class MessageMediaTable {
+    let valueBox: ValueBox
+    let tableId: Int32
+    
+    let mediaCleanupTable: MediaCleanupTable
+    
+    init(valueBox: ValueBox, tableId: Int32, mediaCleanupTable: MediaCleanupTable) {
+        self.valueBox = valueBox
+        self.tableId = tableId
+        self.mediaCleanupTable = mediaCleanupTable
+    }
+
+    func key(id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey {
+        key.setInt32(0, value: id.namespace)
+        key.setInt64(4, value: id.id)
+        return key
+    }
+    
+    func get(id: MediaId) -> Media? {
+        if let value = self.valueBox.get(self.tableId, key: self.key(id)) {
+            var type: Int8 = 0
+            value.read(&type, offset: 0, length: 1)
+            if type == MediaEntryType.Direct.rawValue {
+                var dataLength: Int32 = 0
+                value.read(&dataLength, offset: 0, length: 4)
+                if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                    return media
+                }
+            }
+        }
+        return nil
+    }
+    
+    func set(media: Media, index: MessageIndex, messageHistoryTable: MessageHistoryTable, sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: Encoder = Encoder()) -> InsertMediaResult {
+        if let id = media.id {
+            if let value = self.valueBox.get(self.tableId, key: self.key(id)) {
+                var type: Int8 = 0
+                value.read(&type, offset: 0, length: 1)
+                if type == MediaEntryType.Direct.rawValue {
+                    var dataLength: Int32 = 0
+                    value.read(&dataLength, offset: 0, length: 4)
+                    value.skip(Int(dataLength))
+                    
+                    sharedWriteBuffer.reset()
+                    sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset)
+                    
+                    var messageReferenceCount: Int32 = 0
+                    value.read(&messageReferenceCount, offset: 0, length: 4)
+                    messageReferenceCount++
+                    sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                    
+                    self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                    
+                    return .Reference
+                } else if type == MediaEntryType.MessageReference.rawValue {
+                    var idPeerId: Int64 = 0
+                    var idNamespace: Int32 = 0
+                    var idId: Int32 = 0
+                    var idTimestamp: Int32 = 0
+                    value.read(&idPeerId, offset: 0, length: 8)
+                    value.read(&idNamespace, offset: 0, length: 4)
+                    value.read(&idId, offset: 0, length: 4)
+                    value.read(&idTimestamp, offset: 0, length: 4)
+                    
+                    let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                    if referencedMessageIndex == index {
+                        return .Embed(media)
+                    }
+                    
+                    if let media = messageHistoryTable.unembedMedia(referencedMessageIndex, id: id) {
+                        sharedWriteBuffer.reset()
+                        var directType: Int8 = MediaEntryType.Direct.rawValue
+                        sharedWriteBuffer.write(&directType, offset: 0, length: 1)
+                        
+                        sharedEncoder.reset()
+                        sharedEncoder.encodeRootObject(media)
+                        let mediaBuffer = sharedEncoder.memoryBuffer()
+                        var mediaBufferLength = Int32(mediaBuffer.length)
+                        sharedWriteBuffer.write(&mediaBufferLength, offset: 0, length: 4)
+                        sharedWriteBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length)
+                        
+                        var messageReferenceCount: Int32 = 2
+                        sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                        
+                        self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                    }
+                    
+                    return .Reference
+                } else {
+                    return .Embed(media)
+                }
+            } else {
+                sharedWriteBuffer.reset()
+                var type: Int8 = MediaEntryType.MessageReference.rawValue
+                sharedWriteBuffer.write(&type, offset: 0, length: 1)
+                var idPeerId: Int64 = index.id.peerId.toInt64()
+                var idNamespace: Int32 = index.id.namespace
+                var idId: Int32 = index.id.id
+                var idTimestamp: Int32 = index.timestamp
+                sharedWriteBuffer.write(&idPeerId, offset: 0, length: 8)
+                sharedWriteBuffer.write(&idNamespace, offset: 0, length: 4)
+                sharedWriteBuffer.write(&idId, offset: 0, length: 4)
+                sharedWriteBuffer.write(&idTimestamp, offset: 0, length: 4)
+                
+                self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                
+                return .Embed(media)
+            }
+        } else {
+            return .Embed(media)
+        }
+    }
+    
+    func removeReference(id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) {
+        if let value = self.valueBox.get(self.tableId, key: self.key(id)) {
+            var type: Int8 = 0
+            value.read(&type, offset: 0, length: 1)
+            if type == MediaEntryType.Direct.rawValue {
+                var dataLength: Int32 = 0
+                value.read(&dataLength, offset: 0, length: 4)
+                let mediaOffset = value.offset
+                value.skip(Int(dataLength))
+                
+                sharedWriteBuffer.reset()
+                sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset)
+                
+                var messageReferenceCount: Int32 = 0
+                value.read(&messageReferenceCount, offset: 0, length: 4)
+                messageReferenceCount -= 1
+                sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                
+                if messageReferenceCount <= 0 {
+                    if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + mediaOffset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                        self.mediaCleanupTable.add(media)
+                    }
+                    self.valueBox.remove(self.tableId, key: self.key(id))
+                } else {
+                    self.valueBox.set(self.tableId, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                }
+            } else if type == MediaEntryType.MessageReference.rawValue {
+                self.valueBox.remove(self.tableId, key: self.key(id))
+            }
+        }
+    }
+    
+    func removeEmbeddedMedia(media: Media) {
+        if let id = media.id {
+            self.valueBox.remove(self.tableId, key: self.key(id))
+        }
+        self.mediaCleanupTable.add(media)
+    }
+    
+    func debugList() -> [DebugMediaEntry] {
+        var entries: [DebugMediaEntry] = []
+        
+        let upperBoundKey = ValueBoxKey(length: 8 + 4)
+        memset(upperBoundKey.memory, 0xff, 8 + 4)
+        self.valueBox.range(self.tableId, start: ValueBoxKey(length: 0), end: upperBoundKey, values: { key, value in
+            var type: Int8 = 0
+            value.read(&type, offset: 0, length: 1)
+            if type == MediaEntryType.Direct.rawValue {
+                var dataLength: Int32 = 0
+                value.read(&dataLength, offset: 0, length: 4)
+                if let media = Decoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                    
+                    value.skip(Int(dataLength))
+                    
+                    var messageReferenceCount: Int32 = 0
+                    value.read(&messageReferenceCount, offset: 0, length: 4)
+                    
+                    entries.append(.Direct(media, Int(messageReferenceCount)))
+                }
+            } else if type == MediaEntryType.MessageReference.rawValue {
+                var idPeerId: Int64 = 0
+                var idNamespace: Int32 = 0
+                var idId: Int32 = 0
+                var idTimestamp: Int32 = 0
+                value.read(&idPeerId, offset: 0, length: 8)
+                value.read(&idNamespace, offset: 0, length: 4)
+                value.read(&idId, offset: 0, length: 4)
+                value.read(&idTimestamp, offset: 0, length: 4)
+                
+                let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                
+                entries.append(.MessageReference(referencedMessageIndex))
+            }
+            
+            return true
+        }, limit: 1000)
+        
+        return entries
+    }
+}

Postbox/MessageView.swift

@@ -1,428 +0,0 @@
-import Foundation
-
-public final class MutableMessageView: CustomStringConvertible {
-    public struct RemoveContext {
-        var invalidEarlier: Set<MessageId.Namespace>
-        var invalidLater: Set<MessageId.Namespace>
-        var removedMessages: Bool
-        
-        init() {
-            self.invalidEarlier = []
-            self.invalidLater = []
-            self.removedMessages = false
-        }
-        
-        func empty() -> Bool {
-            return !self.removedMessages && self.invalidEarlier.count == 0 && self.invalidLater.count == 0
-        }
-    }
-    
-    let namespaces: [MessageId.Namespace]
-    let count: Int
-    var earlier: [MessageId.Namespace : RenderedMessage] = [:]
-    var later: [MessageId.Namespace : RenderedMessage] = [:]
-    var messages: [RenderedMessage]
-    
-    public init(namespaces: [MessageId.Namespace], count: Int, earlier: [MessageId.Namespace : RenderedMessage], messages: [RenderedMessage], later: [MessageId.Namespace : RenderedMessage]) {
-        self.namespaces = namespaces
-        self.count = count
-        self.earlier = earlier
-        self.later = later
-        self.messages = messages
-    }
-    
-    public func add(message: RenderedMessage) -> Bool {
-        if self.messages.count == 0 {
-            self.messages.append(message)
-            return true
-        } else {
-            let first = MessageIndex(self.messages[self.messages.count - 1].message)
-            let last = MessageIndex(self.messages[0].message)
-            
-            var next: MessageIndex?
-            for namespace in self.namespaces {
-                if let message = later[namespace] {
-                    let messageIndex = MessageIndex(message.message)
-                    if next == nil || messageIndex < next! {
-                        next = messageIndex
-                    }
-                }
-            }
-            
-            let index = MessageIndex(message.message)
-            
-            if index < last {
-                let earlierMessage = self.earlier[message.message.id.namespace]
-                if earlierMessage == nil || earlierMessage!.message.id.id < message.message.id.id {
-                    if self.messages.count < self.count {
-                        self.messages.insert(message, atIndex: 0)
-                    } else {
-                        self.earlier[message.message.id.namespace] = message
-                    }
-                    return true
-                } else {
-                    return false
-                }
-            } else if index > first {
-                if next != nil && index > next! {
-                    let laterMessage = self.later[message.message.id.namespace]
-                    if laterMessage == nil || laterMessage!.message.id.id > message.message.id.id {
-                        if self.messages.count < self.count {
-                            self.messages.append(message)
-                        } else {
-                            self.later[message.message.id.namespace] = message
-                        }
-                        return true
-                    } else {
-                        return false
-                    }
-                } else {
-                    self.messages.append(message)
-                    if self.messages.count > self.count {
-                        let earliest = self.messages[0]
-                        self.earlier[earliest.message.id.namespace] = earliest
-                        self.messages.removeAtIndex(0)
-                    }
-                    return true
-                }
-            } else if index != last && index != first {
-                var i = self.messages.count
-                while i >= 1 {
-                    if MessageIndex(self.messages[i - 1].message) < index {
-                        break
-                    }
-                    i--
-                }
-                self.messages.insert(message, atIndex: i)
-                if self.messages.count > self.count {
-                    let earliest = self.messages[0]
-                    self.earlier[earliest.message.id.namespace] = earliest
-                    self.messages.removeAtIndex(0)
-                }
-                return true
-            } else {
-                return false
-            }
-        }
-    }
-    
-    public func remove(ids: Set<MessageId>, context: RemoveContext? = nil) -> RemoveContext {
-        var updatedContext = RemoveContext()
-        if let context = context {
-            updatedContext = context
-        }
-        
-        for (_, message) in self.earlier {
-            if ids.contains(message.message.id) {
-                updatedContext.invalidEarlier.insert(message.message.id.namespace)
-            }
-        }
-
-        for (_, message) in self.later {
-            if ids.contains(message.message.id) {
-                updatedContext.invalidLater.insert(message.message.id.namespace)
-            }
-        }
-        
-        if self.messages.count != 0 {
-            var i = self.messages.count - 1
-            while i >= 0 {
-                if ids.contains(self.messages[i].message.id) {
-                    self.messages.removeAtIndex(i)
-                    updatedContext.removedMessages = true
-                }
-                i--
-            }
-        }
-        
-        return updatedContext
-    }
-    
-    public func complete(context: RemoveContext, fetchEarlier: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage], fetchLater: (MessageId.Namespace, MessageId.Id?, Int) -> [RenderedMessage]) {
-        if context.removedMessages {
-            var addedMessages: [RenderedMessage] = []
-            
-            var latestAnchor: MessageIndex?
-            if let lastMessage = self.messages.last {
-                latestAnchor = MessageIndex(lastMessage.message)
-            }
-            
-            if latestAnchor == nil {
-                for (_, message) in self.later {
-                    let messageIndex = MessageIndex(message.message)
-                    if latestAnchor == nil || latestAnchor! > messageIndex {
-                        latestAnchor = messageIndex
-                    }
-                }
-            }
-            
-            for namespace in self.namespaces {
-                if let later = self.later[namespace] {
-                    addedMessages += fetchLater(namespace, later.message.id.id - 1, self.count)
-                }
-                if let earlier = self.earlier[namespace] {
-                    addedMessages += fetchEarlier(namespace, earlier.message.id.id + 1, self.count)
-                }
-            }
-            
-            addedMessages += self.messages
-            addedMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) })
-            var i = addedMessages.count - 1
-            while i >= 1 {
-                if addedMessages[i].message.id == addedMessages[i - 1].message.id {
-                    addedMessages.removeAtIndex(i)
-                }
-                i--
-            }
-            self.messages = []
-            
-            var anchorIndex = addedMessages.count - 1
-            if let latestAnchor = latestAnchor {
-                var i = addedMessages.count - 1
-                while i >= 0 {
-                    if MessageIndex(addedMessages[i].message) <= latestAnchor {
-                        anchorIndex = i
-                        break
-                    }
-                    i--
-                }
-            }
-            
-            self.later.removeAll(keepCapacity: true)
-            if anchorIndex + 1 < addedMessages.count {
-                for namespace in self.namespaces {
-                    var i = anchorIndex + 1
-                    while i < addedMessages.count {
-                        if addedMessages[i].message.id.namespace == namespace {
-                            self.later[namespace] = addedMessages[i]
-                            break
-                        }
-                        i++
-                    }
-                }
-            }
-            
-            i = anchorIndex
-            while i >= 0 && i > anchorIndex - self.count {
-                self.messages.insert(addedMessages[i], atIndex: 0)
-                i--
-            }
-            
-            self.earlier.removeAll(keepCapacity: true)
-            if anchorIndex - self.count >= 0 {
-                for namespace in self.namespaces {
-                    i = anchorIndex - self.count
-                    while i >= 0 {
-                        if addedMessages[i].message.id.namespace == namespace {
-                            self.earlier[namespace] = addedMessages[i]
-                            break
-                        }
-                        i--
-                    }
-                }
-            }
-        }
-        else {
-            for namespace in context.invalidEarlier {
-                var earlyId: MessageId.Id?
-                var i = 0
-                while i < self.messages.count {
-                    if self.messages[i].message.id.namespace == namespace {
-                        earlyId = self.messages[i].message.id.id
-                        break
-                    }
-                    i++
-                }
-                
-                let earlierMessages = fetchEarlier(namespace, earlyId, 1)
-                if earlierMessages.count == 0 {
-                    self.earlier.removeValueForKey(namespace)
-                } else {
-                    self.earlier[namespace] = earlierMessages[0]
-                }
-            }
-            
-            for namespace in context.invalidLater {
-                var lateId: MessageId.Id?
-                var i = self.messages.count - 1
-                while i >= 0 {
-                    if self.messages[i].message.id.namespace == namespace {
-                        lateId = self.messages[i].message.id.id
-                        break
-                    }
-                    i--
-                }
-                
-                let laterMessages = fetchLater(namespace, lateId, 1)
-                if laterMessages.count == 0 {
-                    self.later.removeValueForKey(namespace)
-                } else {
-                    self.later[namespace] = laterMessages[0]
-                }
-            }
-        }
-    }
-    
-    
-    public func incompleteMessages() -> [Message] {
-        var result: [Message] = []
-        
-        for (_, message) in self.earlier {
-            if message.incomplete {
-                result.append(message.message)
-            }
-        }
-        for (_, message) in self.later {
-            if message.incomplete {
-                result.append(message.message)
-            }
-        }
-        
-        for message in self.messages {
-            if message.incomplete {
-                result.append(message.message)
-            }
-        }
-        
-        return result
-    }
-    
-    public func completeMessages(messages: [MessageId : RenderedMessage]) {
-        var earlier = self.earlier
-        for (namespace, message) in self.earlier {
-            if let message = messages[message.message.id] {
-                earlier[namespace] = message
-            }
-        }
-        self.earlier = earlier
-        
-        var later = self.later
-        for (namespace, message) in self.later {
-            if let message = messages[message.message.id] {
-                later[namespace] = message
-            }
-        }
-        self.later = later
-        
-        var i = 0
-        while i < self.messages.count {
-            if let message = messages[self.messages[i].message.id] {
-                self.messages[i] = message
-            }
-            i++
-        }
-    }
-    
-    public var description: String {
-        var string = ""
-        string += "...("
-        var first = true
-        for namespace in self.namespaces {
-            if let value = self.earlier[namespace] {
-                if first {
-                    first = false
-                } else {
-                    string += ", "
-                }
-                string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)"
-            }
-        }
-        string += ") —— "
-        
-        string += "["
-        first = true
-        for message in self.messages {
-            if first {
-                first = false
-            } else {
-                string += ", "
-            }
-            string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)"
-        }
-        string += "]"
-        
-        string += " —— ("
-        first = true
-        for namespace in self.namespaces {
-            if let value = self.later[namespace] {
-                if first {
-                    first = false
-                } else {
-                    string += ", "
-                }
-                string += "\(namespace): \(value.message.id.id)—\(value.message.timestamp)"
-            }
-        }
-        string += ")..."
-        
-        return string
-    }
-}
-
-public final class MessageView: CustomStringConvertible {
-    public let hasEarlier: Bool
-    private let earlierIds: [MessageIndex]
-    public let hasLater: Bool
-    private let laterIds: [MessageIndex]
-    public let messages: [RenderedMessage]
-    
-    init(_ mutableView: MutableMessageView) {
-        self.hasEarlier = mutableView.earlier.count != 0
-        self.hasLater = mutableView.later.count != 0
-        self.messages = mutableView.messages
-        
-        var earlierIds: [MessageIndex] = []
-        for (_, message) in mutableView.earlier {
-            earlierIds.append(MessageIndex(message.message))
-        }
-        self.earlierIds = earlierIds
-        
-        var laterIds: [MessageIndex] = []
-        for (_, message) in mutableView.later {
-            laterIds.append(MessageIndex(message.message))
-        }
-        self.laterIds = laterIds
-    }
-    
-    public var description: String {
-        var string = ""
-        if self.hasEarlier {
-            string += "more("
-            var first = true
-            for id in self.earlierIds {
-                if first {
-                    first = false
-                } else {
-                    string += ", "
-                }
-                string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)"
-            }
-            string += ") "
-        }
-        string += "["
-        var first = true
-        for message in self.messages {
-            if first {
-                first = false
-            } else {
-                string += ", "
-            }
-            string += "\(message.message.id.namespace): \(message.message.id.id)—\(message.message.timestamp)"
-        }
-        string += "]"
-        if self.hasLater {
-            string += " more("
-            var first = true
-            for id in self.laterIds {
-                if first {
-                    first = false
-                } else {
-                    string += ", "
-                }
-                string += "\(id.id.namespace): \(id.id.id)—\(id.timestamp)"
-            }
-            string += ")"
-        }
-        return string
-    }
-}

Postbox/Peer.swift

@@ -59,11 +59,13 @@ public struct PeerId: Hashable, CustomStringConvertible, Comparable {
     }
     
     public init(_ buffer: ReadBuffer) {
-        self.namespace = 0
-        self.id = 0
         
-        memcpy(&self.namespace, buffer.memory, 4)
+        var namespace: Int32 = 0
-        memcpy(&self.id, buffer.memory + 4, 4)
+        var id: Int32 = 0
+        memcpy(&namespace, buffer.memory, 4)
+        self.namespace = namespace
+        memcpy(&id, buffer.memory + 4, 4)
+        self.id = id
     }
     
     public func encodeToBuffer(buffer: WriteBuffer) {

Postbox/Postbox.swift

@@ -48,13 +48,13 @@ public final class Modifier<State: PostboxState> {
 public final class Postbox<State: PostboxState> {
     private let basePath: String
     private let messageNamespaces: [MessageId.Namespace]
-    private let absoluteIndexedMessageNamespaces: [MessageId.Namespace]
+    private let absoluteIndexedMessageNamespace: MessageId.Namespace
     
     private let queue = Queue(name: "org.telegram.postbox.Postbox")
     private var valueBox: ValueBox!
     
-    private var peerMessageViews: [PeerId : Bag<(MutableMessageView, Pipe<MessageView>)>] = [:]
+    private var peerMessageHistoryViews: [PeerId : Bag<(MutableMessageHistoryView, Pipe<MessageHistoryView>)>] = [:]
-    private var deferredMessageViewsToUpdate: [(MutableMessageView, Pipe<MessageView>)] = []
+    private var deferredMessageHistoryViewsToUpdate: [(MutableMessageHistoryView, Pipe<MessageHistoryView>)] = []
     private var peerViews: Bag<(MutablePeerView, Pipe<PeerView>)> = Bag()
     private var deferredPeerViewsToUpdate: [(MutablePeerView, Pipe<PeerView>)] = []
     private var peerPipes: [PeerId : Pipe<Peer>] = [:]
@@ -63,10 +63,14 @@ public final class Postbox<State: PostboxState> {
     
     public let mediaBox: MediaBox
     
-    public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespaces: [MessageId.Namespace]) {
+    public init(basePath: String, messageNamespaces: [MessageId.Namespace], absoluteIndexedMessageNamespace: MessageId.Namespace?) {
         self.basePath = basePath
         self.messageNamespaces = messageNamespaces
-        self.absoluteIndexedMessageNamespaces = absoluteIndexedMessageNamespaces
+        if let absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace {
+            self.absoluteIndexedMessageNamespace = absoluteIndexedMessageNamespace
+        } else {
+            self.absoluteIndexedMessageNamespace = MessageId.Namespace.max
+        }
         self.mediaBox = MediaBox(basePath: self.basePath + "/media")
         self.openDatabase()
     }
@@ -83,7 +87,7 @@ public final class Postbox<State: PostboxState> {
             self.valueBox = SqliteValueBox(basePath: self.basePath + "/db")
             //self.valueBox = LmdbValueBox(basePath: self.basePath + "/db")
             var userVersion: Int32 = 0
-            let currentUserVersion: Int32 = 3
+            let currentUserVersion: Int32 = 4
             
             if let value = self.valueBox.get(Table_Meta.id, key: Table_Meta.key()) {
                 value.read(&userVersion, offset: 0, length: 4)
@@ -166,123 +170,11 @@ public final class Postbox<State: PostboxState> {
     }
     
     private func addMessages(messages: [Message], medias: [Media]) {
-        let encoder = Encoder()
         
-        for (peerId, peerMessages) in messagesGroupedByPeerId(messages) {
-            var maxMessage: (MessageIndex, Message)?
-            
-            var messageIds: [MessageId] = []
-            var seenMessageIds = Set<MessageId>()
-            for message in peerMessages {
-                if !seenMessageIds.contains(message.id) {
-                    seenMessageIds.insert(message.id)
-                    messageIds.append(message.id)
-                }
-            }
-            
-            var existingMessageIds = Set<MessageId>()
-            
-            let existingMessageKey = Table_Message.emptyKey()
-            existingMessageKey.setInt64(0, value: peerId.toInt64())
-            for id in messageIds {
-                if self.valueBox.exists(Table_Message.id, key: Table_Message.key(id, key: existingMessageKey)) {
-                    existingMessageIds.insert(id)
-                }
-            }
-            
-            var addedMessages: [Message] = []
-            
-            let mediaMessageIdKey = Table_Media_MessageIds.emptyKey()
-            for message in peerMessages {
-                if existingMessageIds.contains(message.id) {
-                    continue
-                }
-                existingMessageIds.insert(message.id)
-                addedMessages.append(message)
-                
-                let index = MessageIndex(message)
-                if maxMessage == nil || index > maxMessage!.0 {
-                    maxMessage = (index, message)
-                }
-                
-                encoder.reset()
-                encoder.encodeRootObject(message)
-                
-                for id in message.mediaIds {
-                    self.valueBox.set(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(id, messageId: message.id, key: mediaMessageIdKey), value: MemoryBuffer())
-                }
-                
-                self.valueBox.set(Table_Message.id, key: Table_Message.key(message.id), value: Table_Message.set(message))
-                
-                let absoluteKey = Table_AbsoluteMessageId.emptyKey()
-                if self.absoluteIndexedMessageNamespaces.contains(message.id.namespace) {
-                    self.valueBox.set(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(message.id.id, key: absoluteKey), value: Table_AbsoluteMessageId.set(message.id))
-                }
-            }
-            
-            if let relatedViews = self.peerMessageViews[peerId] {
-                for record in relatedViews.copyItems() {
-                    var updated = false
-                    for message in addedMessages {
-                        if record.0.add(RenderedMessage(message: message)) {
-                            updated = true
-                        }
-                    }
-                    
-                    if updated {
-                        self.deferMessageViewUpdate(record.0, pipe: record.1)
-                    }
-                }
-            }
-            
-            if let maxMessage = maxMessage {
-                self.updatePeerEntry(peerId, message: RenderedMessage(message: maxMessage.1))
-            }
-        }
-        
-        var existingMediaIds = Set<MediaId>()
-        
-        let existingMediaKey = Table_Media.emptyKey()
-        for media in medias {
-            if let id = media.id {
-                if self.valueBox.exists(Table_Media.id, key: Table_Media.key(id, key: existingMediaKey)) {
-                    existingMediaIds.insert(id)
-                }
-            }
-        }
-        
-        let mediaKey = Table_Media.emptyKey()
-        for media in medias {
-            if let id = media.id {
-                if existingMediaIds.contains(id) {
-                    continue
-                }
-                existingMediaIds.insert(id)
-
-                self.valueBox.set(Table_Media.id, key: Table_Media.key(id, key: mediaKey), value: Table_Media.set(media))
-            }
-        }
     }
     
     private func mediaWithIds(ids: [MediaId]) -> [MediaId : Media] {
-        if ids.count == 0 {
         return [:]
-        } else {
-            var result: [MediaId : Media] = [:]
-            
-            let mediaKey = Table_Media.emptyKey()
-            for id in ids {
-                if let value = self.valueBox.get(Table_Media.id, key: Table_Media.key(id, key: mediaKey)) {
-                    if let media = Table_Media.get(value) {
-                        result[id] = media
-                    } else {
-                        print("can't parse media")
-                    }
-                }
-            }
-            
-            return result
-        }
     }
     
     var cachedPeers: [PeerId : Peer] = [:]
@@ -338,19 +230,19 @@ public final class Postbox<State: PostboxState> {
         }
     }
     
-    private func deferMessageViewUpdate(view: MutableMessageView, pipe: Pipe<MessageView>) {
+    private func deferMessageHistoryViewUpdate(view: MutableMessageHistoryView, pipe: Pipe<MessageHistoryView>) {
         var i = 0
         var found = false
-        while i < self.deferredMessageViewsToUpdate.count {
+        while i < self.deferredMessageHistoryViewsToUpdate.count {
-            if self.deferredMessageViewsToUpdate[i].1 === pipe {
+            if self.deferredMessageHistoryViewsToUpdate[i].1 === pipe {
-                self.deferredMessageViewsToUpdate[i] = (view, pipe)
+                self.deferredMessageHistoryViewsToUpdate[i] = (view, pipe)
                 found = true
                 break
             }
             i++
         }
         if !found {
-            self.deferredMessageViewsToUpdate.append((view, pipe))
+            self.deferredMessageHistoryViewsToUpdate.append((view, pipe))
         }
     }
     
@@ -371,10 +263,10 @@ public final class Postbox<State: PostboxState> {
             entry.1.putNext(PeerView(entry.0))
         }
         
-        let deferredMessageViewsToUpdate = self.deferredMessageViewsToUpdate
+        let deferredMessageHistoryViewsToUpdate = self.deferredMessageHistoryViewsToUpdate
-        self.deferredMessageViewsToUpdate.removeAll()
+        self.deferredMessageHistoryViewsToUpdate.removeAll()
         
-        for entry in deferredMessageViewsToUpdate {
+        for entry in deferredMessageHistoryViewsToUpdate {
             let viewRenderedMessages = self.renderedMessages(entry.0.incompleteMessages())
             if viewRenderedMessages.count != 0 {
                 var viewRenderedMessagesDict: [MessageId : RenderedMessage] = [:]
@@ -384,7 +276,7 @@ public final class Postbox<State: PostboxState> {
                 entry.0.completeMessages(viewRenderedMessagesDict)
             }
             
-            entry.1.putNext(MessageView(entry.0))
+            entry.1.putNext(MessageHistoryView(entry.0))
         }
     }
     
@@ -458,10 +350,10 @@ public final class Postbox<State: PostboxState> {
         
         var result: [MessageId] = []
         
-        let key = Table_AbsoluteMessageId.emptyKey()
+        let key = Table_GlobalMessageId.emptyKey()
         for id in ids {
-            if let value = self.valueBox.get(Table_AbsoluteMessageId.id, key: Table_AbsoluteMessageId.key(id, key: key)) {
+            if let value = self.valueBox.get(Table_GlobalMessageId.id, key: Table_GlobalMessageId.key(id, key: key)) {
-                result.append(Table_AbsoluteMessageId.get(id, value: value))
+                result.append(Table_GlobalMessageId.get(id, value: value))
             }
         }
         
@@ -469,55 +361,7 @@ public final class Postbox<State: PostboxState> {
     }
     
     private func deleteMessagesWithIds(ids: [MessageId]) {
-        for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) {
-            if let relatedViews = self.peerMessageViews[peerId] {
-                for (view, pipe) in relatedViews.copyItems() {
-                    let context = view.remove(Set<MessageId>(messageIds))
-                    if !context.empty() {
-                        view.complete(context, fetchEarlier: self.fetchMessagesRelative(peerId, earlier: true), fetchLater: self.fetchMessagesRelative(peerId, earlier: false))
-                        self.deferMessageViewUpdate(view, pipe: pipe)
-                    }
-                }
-            }
-        }
         
-        var touchedMediaIds = Set<MediaId>()
-        
-        let removeMediaMessageIdKey = Table_Media_MessageIds.emptyKey()
-        for (peerId, messageIds) in messageIdsGroupedByPeerId(ids) {
-            let messageKey = Table_Message.emptyKey()
-            for id in messageIds {
-                if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(id, key: messageKey)) {
-                    if let message = Table_Message.get(value) {
-                        for mediaId in message.mediaIds {
-                            touchedMediaIds.insert(mediaId)
-                            self.valueBox.remove(Table_Media_MessageIds.id, key: Table_Media_MessageIds.key(mediaId, messageId: message.id, key: removeMediaMessageIdKey))
-                        }
-                    }
-                }
-            }
-
-            for id in messageIds {
-                self.valueBox.remove(Table_Message.id, key: Table_Message.key(id, key: messageKey))
-            }
-            
-            for mediaId in touchedMediaIds {
-                var referenced = false
-                self.valueBox.range(Table_Media_MessageIds.id, start: Table_Media_MessageIds.lowerBoundKey(mediaId), end: Table_Media_MessageIds.upperBoundKey(mediaId), keys: { key in
-                    referenced = true
-                    return false
-                }, limit: 1)
-                
-                if !referenced {
-                    //TODO write to cleanup queue
-                    self.valueBox.remove(Table_Media.id, key: Table_Media.key(mediaId))
-                }
-            }
-            
-            let tail = self.fetchMessagesTail(peerId, count: 1)
-            
-            self.updatePeerEntry(peerId, message: tail.first, replace: true)
-        }
     }
     
     private func updatePeers(peers: [Peer], update: (Peer, Peer) -> Peer) {
@@ -572,9 +416,10 @@ public final class Postbox<State: PostboxState> {
         return Signal { subscriber in
             self.queue.dispatch {
                 //#if DEBUG
-                    let startTime = CFAbsoluteTimeGetCurrent()
+                    //let startTime = CFAbsoluteTimeGetCurrent()
                 //#endif
                 
+                //self.valueBox.beginStats()
                 self.valueBox.begin()
                 let result = f(Modifier(postbox: self))
                 //print("(Postbox modify took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)")
@@ -584,7 +429,8 @@ public final class Postbox<State: PostboxState> {
                 self.valueBox.commit()
                 
                 //#if DEBUG
-                    print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)")
+                    //print("(Postbox commit took \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms)")
+                //self.valueBox.endStats()
                 //#endif
                 
                 self.performDeferredUpdates()
@@ -596,282 +442,6 @@ public final class Postbox<State: PostboxState> {
         }
     }
     
-    private func findAdjacentMessageIds(peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex) -> (MessageId.Id?, MessageId.Id?) {
-        /*var minId: MessageId.Id?
-        var maxId: MessageId.Id?
-        
-        let lowerBoundKey = ValueBoxKey(length: 8 + 4)
-        lowerBoundKey.setInt64(0, value: peerId.toInt64())
-        lowerBoundKey.setInt32(8, value: namespace)
-        
-        let upperBoundKey = ValueBoxKey(length: 8 + 4)
-        upperBoundKey.setInt64(0, value: peerId.toInt64())
-        upperBoundKey.setInt32(8, value: namespace)
-        
-        self.valueBox.range("peer_messages", start: lowerBoundKey, end: upperBoundKey.successor, keys: { key in
-            
-        }, limit: 1)
-        
-        for row in self.database.prepareCached("SELECT MIN(id), MAX(id) FROM peer_messages WHERE peerId = ? AND namespace = ?").run(peerId.toInt64(), Int64(namespace)) {
-            minId = MessageId.Id(row[0] as! Int64)
-            maxId = MessageId.Id(row[1] as! Int64)
-        }
-        
-        if let minId = minId, maxId = maxId {
-            var minTimestamp: Int32!
-            var maxTimestamp: Int32!
-            for row in self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id IN (?, ?)").run(peerId.toInt64(), Int64(namespace), Int64(minId), Int64(maxId)) {
-                let id = Int32(row[0] as! Int64)
-                let timestamp = Int32(row[1] as! Int64)
-                if id == minId {
-                    minTimestamp = timestamp
-                } else {
-                    maxTimestamp = timestamp
-                }
-            }
-            
-            let earlierMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id <= ? LIMIT 1")
-            let laterMidStatement = self.database.prepareCached("SELECT id, timestamp FROM peer_messages WHERE peerId = ? AND namespace = ? AND id >= ? LIMIT 1")
-            
-            func lowerBound(timestamp: Int32) -> MessageId.Id? {
-                var leftId = minId
-                var leftTimestamp = minTimestamp
-                var rightId = maxId
-                var rightTimestamp = maxTimestamp
-                
-                while leftTimestamp <= timestamp && rightTimestamp >= timestamp {
-                    let approximateMiddleId = leftId + (rightId - leftId) / 2
-                    if approximateMiddleId == leftId {
-                        return rightId
-                    }
-                    var middleId: MessageId.Id?
-                    var middleTimestamp: Int32?
-                    for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) {
-                        middleId = MessageId.Id(row[0] as! Int64)
-                        middleTimestamp = Int32(row[1] as! Int64)
-                        break
-                    }
-                    if middleId == leftId {
-                        return rightId
-                    }
-                    
-                    if let middleId = middleId, middleTimestamp = middleTimestamp {
-                        if middleTimestamp >= timestamp {
-                            rightId = middleId
-                            rightTimestamp = middleTimestamp
-                        } else {
-                            leftId = middleId
-                            leftTimestamp = middleTimestamp
-                        }
-                    } else {
-                        return nil
-                    }
-                }
-                
-                return leftId
-            }
-            
-            func upperBound(timestamp: Int32) -> MessageId.Id? {
-                var leftId = minId
-                var leftTimestamp = minTimestamp
-                var rightId = maxId
-                var rightTimestamp = maxTimestamp
-                
-                while leftTimestamp <= timestamp && rightTimestamp >= timestamp {
-                    let approximateMiddleId = leftId + (rightId - leftId) / 2
-                    if approximateMiddleId == leftId {
-                        return leftId
-                    }
-                    var middleId: MessageId.Id?
-                    var middleTimestamp: Int32?
-                    for row in earlierMidStatement.run(peerId.toInt64(), Int64(namespace), Int64(approximateMiddleId)) {
-                        middleId = MessageId.Id(row[0] as! Int64)
-                        middleTimestamp = Int32(row[1] as! Int64)
-                        break
-                    }
-                    if middleId == leftId {
-                        return leftId
-                    }
-                    
-                    if let middleId = middleId, middleTimestamp = middleTimestamp {
-                        if middleTimestamp <= timestamp {
-                            rightId = middleId
-                            rightTimestamp = middleTimestamp
-                        } else {
-                            leftId = middleId
-                            leftTimestamp = middleTimestamp
-                        }
-                    } else {
-                        return nil
-                    }
-                }
-                
-                return rightTimestamp
-            }
-            
-            if index.id.namespace < namespace {
-                let left = upperBound(index.timestamp - 1)
-                let right = lowerBound(index.timestamp)
-                return (left, right)
-            } else {
-                let left = upperBound(index.timestamp)
-                let right = lowerBound(index.timestamp + 1)
-                return (left, right)
-            }
-        } else {
-            return (nil, nil)
-        }*/
-        
-        return (nil, nil)
-    }
-    
-    private func fetchMessagesAround(peerId: PeerId, anchorId: MessageId, count: Int) -> ([RenderedMessage], [MessageId.Namespace : RenderedMessage], [MessageId.Namespace : RenderedMessage]) {
-        var messages: [RenderedMessage] = []
-        
-        messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: anchorId.namespace, id: anchorId.id, count: count + 1)
-        messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: anchorId.namespace, id: anchorId.id - 1, count: count + 1)
-        
-        messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) })
-        var i = messages.count - 1
-        while i >= 1 {
-            if messages[i].message.id == messages[i - 1].message.id {
-                messages.removeAtIndex(i)
-            }
-            i--
-        }
-        
-        if messages.count == 0 {
-            return ([], [:], [:])
-        } else {
-            var index: MessageIndex!
-            for message in messages {
-                if message.message.id == anchorId {
-                    index = MessageIndex(message.message)
-                    break
-                }
-            }
-            if index == nil {
-                var closestId: MessageId.Id = messages[0].message.id.id
-                var closestDistance = abs(closestId - anchorId.id)
-                let closestTimestamp: Int32 = messages[0].message.timestamp
-                for message in messages {
-                    if abs(message.message.id.id - anchorId.id) < closestDistance {
-                        closestId = message.message.id.id
-                        closestDistance = abs(message.message.id.id - anchorId.id)
-                    }
-                }
-                index = MessageIndex(id: MessageId(peerId: peerId, namespace: anchorId.namespace, id: closestId), timestamp: closestTimestamp)
-            }
-            
-            for namespace in self.messageNamespaces {
-                if namespace != anchorId.namespace {
-                    let (left, right) = self.findAdjacentMessageIds(peerId, namespace: namespace, index: index)
-                    if let left = left {
-                        messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: left + 1, count: count + 1)
-                    }
-                    if let right = right {
-                        messages += self.fetchMessagesRelative(peerId, earlier: false)(namespace: namespace, id: right - 1, count: count + 1)
-                    }
-                }
-            }
-            
-            messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) })
-            var i = messages.count - 1
-            while i >= 1 {
-                if messages[i].message.id == messages[i - 1].message.id {
-                    messages.removeAtIndex(i)
-                }
-                i--
-            }
-            
-            var anchorIndex = messages.count / 2
-            i = 0
-            while i < messages.count {
-                if messages[i].message.id == index.id {
-                    anchorIndex = i
-                    break
-                }
-                i++
-            }
-            
-            var filteredMessages: [RenderedMessage] = []
-            var earlier: [MessageId.Namespace : RenderedMessage] = [:]
-            var later: [MessageId.Namespace : RenderedMessage] = [:]
-            
-            i = anchorIndex
-            var j = anchorIndex - 1
-            var leftIndex = j
-            var rightIndex = i
-            
-            while i < messages.count || j >= 0 {
-                if i < messages.count && filteredMessages.count < count {
-                    filteredMessages.append(messages[i])
-                    rightIndex = i
-                }
-                if j >= 0 && filteredMessages.count < count {
-                    filteredMessages.append(messages[j])
-                    leftIndex = j
-                }
-                
-                i++
-                j--
-            }
-            
-            i = leftIndex - 1
-            while i >= 0 {
-                if earlier[messages[i].message.id.namespace] == nil {
-                    earlier[messages[i].message.id.namespace] = messages[i]
-                }
-                i--
-            }
-            
-            i = rightIndex + 1
-            while i < messages.count {
-                if later[messages[i].message.id.namespace] == nil {
-                    later[messages[i].message.id.namespace] = messages[i]
-                }
-                i++
-            }
-            
-            filteredMessages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message) })
-            
-            return (filteredMessages, earlier, later)
-        }
-    }
-    
-    private func fetchMessagesRelative(peerId: PeerId, earlier: Bool)(namespace: MessageId.Namespace, id: MessageId.Id?, count: Int) -> [RenderedMessage] {
-        var messages: [Message] = []
-        
-        let lowerBound = Table_Message.lowerBoundKey(peerId, namespace: namespace)
-        let upperBound = Table_Message.upperBoundKey(peerId, namespace: namespace)
-        
-        let bound: ValueBoxKey
-        if let id = id {
-            bound = Table_Message.key(MessageId(peerId: peerId, namespace: namespace, id: id))
-        } else if earlier {
-            bound = upperBound
-        } else {
-            bound = lowerBound
-        }
-        
-        let values: (ValueBoxKey, ReadBuffer) -> Bool = { _, value in
-            if let message = Table_Message.get(value) {
-                messages.append(message)
-            } else {
-                print("can't parse message")
-            }
-            return true
-        }
-        
-        if earlier {
-            self.valueBox.range(Table_Message.id, start: bound, end: lowerBound, values: values, limit: count)
-        } else {
-            self.valueBox.range(Table_Message.id, start: bound, end: upperBound, values: values, limit: count)
-        }
-        
-        return self.renderedMessages(messages)
-    }
-    
     private func fetchPeerEntryIndicesRelative(earlier: Bool)(index: PeerViewEntryIndex?, count: Int) -> [PeerViewEntryIndex] {
         var entries: [PeerViewEntryIndex] = []
         
@@ -918,7 +488,7 @@ public final class Postbox<State: PostboxState> {
             }
             
             var message: Message?
-            if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex.id)) {
+            if let value = self.valueBox.get(Table_Message.id, key: Table_Message.key(entryIndex.messageIndex)) {
                 message = Table_Message.get(value)
             }
             
@@ -943,123 +513,38 @@ public final class Postbox<State: PostboxState> {
             }
         }
         
-        entries.sortInPlace({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) })
+        //entries.sortInPlace()
         
-        return entries
+        return entries.sort({ PeerViewEntryIndex($0) < PeerViewEntryIndex($1) })
     }
     
     private func renderedMessages(messages: [Message]) -> [RenderedMessage] {
-        if messages.count == 0 {
         return []
     }
     
-        var peerIds = Set<PeerId>()
+    public func tailMessageHistoryViewForPeerId(peerId: PeerId, count: Int) -> Signal<MessageHistoryView, NoError> {
-        var mediaIds = Set<MediaId>()
-        
-        for message in messages {
-            for peerId in message.peerIds {
-                peerIds.insert(peerId)
-            }
-            for mediaId in message.mediaIds {
-                mediaIds.insert(mediaId)
-            }
-        }
-        
-        var arrayPeerIds: [PeerId] = []
-        for id in peerIds {
-            arrayPeerIds.append(id)
-        }
-        let peers = self.peersWithIds(arrayPeerIds)
-        
-        var arrayMediaIds: [MediaId] = []
-        for id in mediaIds {
-            arrayMediaIds.append(id)
-        }
-        let medias = self.mediaWithIds(arrayMediaIds)
-        
-        var result: [RenderedMessage] = []
-        
-        for message in messages {
-            if message.peerIds.count == 0 && message.mediaIds.count == 0 {
-                result.append(RenderedMessage(message: message, peers: [], media: []))
-            } else {
-                var messagePeers: [Peer] = []
-                for id in message.peerIds {
-                    if let peer = peers[id] {
-                        messagePeers.append(peer)
-                    }
-                }
-                
-                var messageMedia: [Media] = []
-                for id in message.mediaIds {
-                    if let media = medias[id] {
-                        messageMedia.append(media)
-                    }
-                }
-                
-                result.append(RenderedMessage(message: message, peers: messagePeers, media: messageMedia))
-            }
-        }
-        
-        return result
-    }
-    
-    private func fetchMessagesTail(peerId: PeerId, count: Int) -> [RenderedMessage] {
-        var messages: [RenderedMessage] = []
-        
-        for namespace in self.messageNamespaces {
-            messages += self.fetchMessagesRelative(peerId, earlier: true)(namespace: namespace, id: nil, count: count)
-        }
-        
-        messages.sortInPlace({ MessageIndex($0.message) < MessageIndex($1.message)})
-        
-        return messages
-    }
-    
-    public func tailMessageViewForPeerId(peerId: PeerId, count: Int) -> Signal<MessageView, NoError> {
         return Signal { subscriber in
             let startTime = CFAbsoluteTimeGetCurrent()
             
             let disposable = MetaDisposable()
             
             self.queue.dispatch {
-                let tail = self.fetchMessagesTail(peerId, count: count + 1)
                 
-                print("tailMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms")
+                print("tailMessageHistoryViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms")
                 
-                var messages: [RenderedMessage] = []
+                let mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil)
-                var i = tail.count - 1
+                let record = (mutableView, Pipe<MessageHistoryView>())
-                while i >= 0 && i >= tail.count - count {
-                    messages.insert(tail[i], atIndex: 0)
-                    i--
-                }
                 
-                var earlier: [MessageId.Namespace : RenderedMessage] = [:]
+                let index: Bag<(MutableMessageHistoryView, Pipe<MessageHistoryView>)>.Index
-                
+                if let bag = self.peerMessageHistoryViews[peerId] {
-                for namespace in self.messageNamespaces {
-                    var i = tail.count - count - 1
-                    while i >= 0 {
-                        if tail[i].message.id.namespace == namespace {
-                            earlier[namespace] = tail[i]
-                            break
-                        }
-                        i--
-                    }
-                }
-                
-                let mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:])
-                let record = (mutableView, Pipe<MessageView>())
-                
-                let index: Bag<(MutableMessageView, Pipe<MessageView>)>.Index
-                if let bag = self.peerMessageViews[peerId] {
                     index = bag.add(record)
                 } else {
-                    let bag = Bag<(MutableMessageView, Pipe<MessageView>)>()
+                    let bag = Bag<(MutableMessageHistoryView, Pipe<MessageHistoryView>)>()
                     index = bag.add(record)
-                    self.peerMessageViews[peerId] = bag
+                    self.peerMessageHistoryViews[peerId] = bag
                 }
                 
-                subscriber.putNext(MessageView(mutableView))
+                subscriber.putNext(MessageHistoryView(mutableView))
                 
                 let pipeDisposable = record.1.signal().start(next: { next in
                     subscriber.putNext(next)
@@ -1070,7 +555,7 @@ public final class Postbox<State: PostboxState> {
                     
                     if let strongSelf = self {
                         strongSelf.queue.dispatch {
-                            if let bag = strongSelf.peerMessageViews[peerId] {
+                            if let bag = strongSelf.peerMessageHistoryViews[peerId] {
                                 bag.remove(index)
                             }
                         }
@@ -1083,58 +568,31 @@ public final class Postbox<State: PostboxState> {
         }
     }
     
-    public func aroundMessageViewForPeerId(peerId: PeerId, id: MessageId, count: Int) -> Signal<MessageView, NoError> {
+    public func aroundMessageHistoryViewForPeerId(peerId: PeerId, index: MessageIndex, count: Int) -> Signal<MessageHistoryView, NoError> {
         return Signal { subscriber in
             let disposable = MetaDisposable()
             
             self.queue.dispatch {
-                let mutableView: MutableMessageView
+                let mutableView: MutableMessageHistoryView
                 
                 let startTime = CFAbsoluteTimeGetCurrent()
                 
-                let around = self.fetchMessagesAround(peerId, anchorId: id, count: count)
+                mutableView = MutableMessageHistoryView(count: count, earlierMessage: nil, messages: [], laterMessage: nil)
-                if around.0.count == 0 {
-                    let tail = self.fetchMessagesTail(peerId, count: count + 1)
-                    
-                    var messages: [RenderedMessage] = []
-                    var i = tail.count - 1
-                    while i >= 0 && i >= tail.count - count {
-                        messages.insert(tail[i], atIndex: 0)
-                        i--
-                    }
-                    
-                    var earlier: [MessageId.Namespace : RenderedMessage] = [:]
-                    
-                    for namespace in self.messageNamespaces {
-                        var i = tail.count - count - 1
-                        while i >= 0 {
-                            if tail[i].message.id.namespace == namespace {
-                                earlier[namespace] = tail[i]
-                                break
-                            }
-                            i--
-                        }
-                    }
-                    
-                    mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: earlier, messages: messages, later: [:])
-                } else {
-                    mutableView = MutableMessageView(namespaces: self.messageNamespaces, count: count, earlier: around.1, messages: around.0, later: around.2)
-                }
                 
                 print("aroundMessageViewForPeerId fetch: \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms")
                 
-                let record = (mutableView, Pipe<MessageView>())
+                let record = (mutableView, Pipe<MessageHistoryView>())
                 
-                let index: Bag<(MutableMessageView, Pipe<MessageView>)>.Index
+                let index: Bag<(MutableMessageHistoryView, Pipe<MessageHistoryView>)>.Index
-                if let bag = self.peerMessageViews[peerId] {
+                if let bag = self.peerMessageHistoryViews[peerId] {
                     index = bag.add(record)
                 } else {
-                    let bag = Bag<(MutableMessageView, Pipe<MessageView>)>()
+                    let bag = Bag<(MutableMessageHistoryView, Pipe<MessageHistoryView>)>()
                     index = bag.add(record)
-                    self.peerMessageViews[peerId] = bag
+                    self.peerMessageHistoryViews[peerId] = bag
                 }
                 
-                subscriber.putNext(MessageView(mutableView))
+                subscriber.putNext(MessageHistoryView(mutableView))
                 
                 let pipeDisposable = record.1.signal().start(next: { next in
                     subscriber.putNext(next)
@@ -1145,7 +603,7 @@ public final class Postbox<State: PostboxState> {
                     
                     if let strongSelf = self {
                         strongSelf.queue.dispatch {
-                            if let bag = strongSelf.peerMessageViews[peerId] {
+                            if let bag = strongSelf.peerMessageHistoryViews[peerId] {
                                 bag.remove(index)
                             }
                         }

Postbox/PostboxTables.swift

@@ -41,33 +41,31 @@ struct Table_Message {
     static let id: Int32 = 4
     
     static func emptyKey() -> ValueBoxKey {
-        return ValueBoxKey(length: 8 + 4 + 4)
+        return ValueBoxKey(length: 8 + 4 + 4 + 4)
     }
     
-    static func lowerBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey {
+    static func lowerBoundKey(peerId: PeerId) -> ValueBoxKey {
-        let key = ValueBoxKey(length: 8 + 4)
+        let key = ValueBoxKey(length: 8)
         key.setInt64(0, value: peerId.toInt64())
-        key.setInt32(8, value: namespace)
         return key
     }
     
-    static func upperBoundKey(peerId: PeerId, namespace: Int32) -> ValueBoxKey {
+    static func upperBoundKey(peerId: PeerId) -> ValueBoxKey {
-        let key = ValueBoxKey(length: 8 + 4)
+        let key = ValueBoxKey(length: 8)
         key.setInt64(0, value: peerId.toInt64())
-        key.setInt32(8, value: namespace)
         return key.successor
     }
     
-    static func key(messageId: MessageId, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey {
+    static func key(index: MessageIndex, key: ValueBoxKey = Table_Message.emptyKey()) -> ValueBoxKey {
-        key.setInt64(0, value: messageId.peerId.toInt64())
+        key.setInt64(0, value: index.id.peerId.toInt64())
-        key.setInt32(8, value: messageId.namespace)
+        key.setInt32(8, value: index.timestamp)
-        key.setInt32(8 + 4, value: messageId.id)
+        key.setInt32(8 + 4, value: index.id.namespace)
+        key.setInt32(8 + 4 + 4, value: index.id.id)
         return key
     }
     
     static func set(message: Message, encoder: Encoder = Encoder()) -> MemoryBuffer {
         encoder.reset()
-        encoder.encodeRootObject(message)
         return encoder.memoryBuffer()
     }
     
@@ -79,14 +77,14 @@ struct Table_Message {
     }
 }
 
-struct Table_AbsoluteMessageId {
+struct Table_GlobalMessageId {
     static let id: Int32 = 5
     
     static func emptyKey() -> ValueBoxKey {
         return ValueBoxKey(length: 4)
     }
     
-    static func key(id: Int32, key: ValueBoxKey = Table_AbsoluteMessageId.emptyKey()) -> ValueBoxKey {
+    static func key(id: Int32, key: ValueBoxKey = Table_GlobalMessageId.emptyKey()) -> ValueBoxKey {
         key.setInt32(0, value: id)
         return key
     }
@@ -272,3 +270,31 @@ struct Table_PeerEntry {
         return index
     }
 }
+
+struct Table_MessageIndex {
+    static let id: Int32 = 11
+    
+    static func emptyKey() -> ValueBoxKey {
+        return ValueBoxKey(length: 8 + 4 + 4)
+    }
+    
+    static func key(id: MessageId, key: ValueBoxKey = Table_MessageIndex.emptyKey()) -> ValueBoxKey {
+        key.setInt64(0, value: id.peerId.toInt64())
+        key.setInt32(8, value: id.namespace)
+        key.setInt32(8 + 4, value: id.id)
+        return key
+    }
+    
+    static func set(index: MessageIndex) -> MemoryBuffer {
+        let buffer = WriteBuffer()
+        var timestamp = index.timestamp
+        buffer.write(&timestamp, offset: 0, length: 4)
+        return buffer.readBufferNoCopy()
+    }
+    
+    static func get(id: MessageId, value: ReadBuffer) -> MessageIndex {
+        var timestamp: Int32 = 0
+        value.read(&timestamp, offset: 0, length: 4)
+        return MessageIndex(id: MessageId(peerId: id.peerId, namespace: id.namespace, id: id.id), timestamp: timestamp)
+    }
+}

Postbox/SqliteValueBox.swift

@@ -65,6 +65,10 @@ public final class SqliteValueBox: ValueBox {
     private var insertStatements: [Int32 : SqlitePreparedStatement] = [:]
     private var deleteStatements: [Int32 : SqlitePreparedStatement] = [:]
     
+    private var readQueryTime: CFAbsoluteTime = 0.0
+    private var writeQueryTime: CFAbsoluteTime = 0.0
+    private var commitTime: CFAbsoluteTime = 0.0
+    
     public init(basePath: String) {
         do {
             try NSFileManager.defaultManager().createDirectoryAtPath(basePath, withIntermediateDirectories: true, attributes: nil)
@@ -91,12 +95,28 @@ public final class SqliteValueBox: ValueBox {
         }
     }
     
+    deinit {
+        self.clearStatements()
+    }
+    
+    public func beginStats() {
+        self.readQueryTime = 0.0
+        self.writeQueryTime = 0.0
+        self.commitTime = 0.0
+    }
+    
+    public func endStats() {
+        print("(SqliteValueBox stats read: \(self.readQueryTime * 1000.0) ms, write: \(self.writeQueryTime * 1000.0) ms, commit: \(self.commitTime * 1000.0) ms")
+    }
+    
     public func begin() {
         self.database.transaction()
     }
     
     public func commit() {
+        let startTime = CFAbsoluteTimeGetCurrent()
         self.database.commit()
+        self.commitTime += CFAbsoluteTimeGetCurrent() - startTime
     }
     
     private func getStatement(table: Int32, key: ValueBoxKey) -> SqlitePreparedStatement {
@@ -372,6 +392,7 @@ public final class SqliteValueBox: ValueBox {
     }
     
     public func get(table: Int32, key: ValueBoxKey) -> ReadBuffer? {
+        let startTime = CFAbsoluteTimeGetCurrent()
         if self.tables.contains(table) {
             let statement = self.getStatement(table, key: key)
             
@@ -384,6 +405,8 @@ public final class SqliteValueBox: ValueBox {
             
             statement.reset()
             
+            self.readQueryTime += CFAbsoluteTimeGetCurrent() - startTime
+            
             return buffer
         }
         
@@ -405,6 +428,8 @@ public final class SqliteValueBox: ValueBox {
         if self.tables.contains(table) {
             let statement: SqlitePreparedStatement
             
+            var startTime = CFAbsoluteTimeGetCurrent()
+            
             if start < end {
                 if limit <= 0 {
                     statement = self.rangeValueAscStatementNoLimit(table, start: start, end: end)
@@ -419,10 +444,20 @@ public final class SqliteValueBox: ValueBox {
                 }
             }
             
+            var currentTime = CFAbsoluteTimeGetCurrent()
+            self.readQueryTime += currentTime - startTime
+            
+            startTime = currentTime
+            
             while statement.step() {
+                startTime = CFAbsoluteTimeGetCurrent()
+                
                 let key = statement.keyAt(0)
                 let value = statement.valueAt(1)
                 
+                currentTime = CFAbsoluteTimeGetCurrent()
+                self.readQueryTime += currentTime - startTime
+                
                 if !values(key, value) {
                     break
                 }
@@ -436,6 +471,8 @@ public final class SqliteValueBox: ValueBox {
         if self.tables.contains(table) {
             let statement: SqlitePreparedStatement
             
+            var startTime = CFAbsoluteTimeGetCurrent()
+            
             if start < end {
                 if limit <= 0 {
                     statement = self.rangeKeyAscStatementNoLimit(table, start: start, end: end)
@@ -450,9 +487,19 @@ public final class SqliteValueBox: ValueBox {
                 }
             }
             
+            var currentTime = CFAbsoluteTimeGetCurrent()
+            self.readQueryTime += currentTime - startTime
+            
+            startTime = currentTime
+            
             while statement.step() {
+                startTime = CFAbsoluteTimeGetCurrent()
+                
                 let key = statement.keyAt(0)
                 
+                currentTime = CFAbsoluteTimeGetCurrent()
+                self.readQueryTime += currentTime - startTime
+                
                 if !keys(key) {
                     break
                 }
@@ -470,6 +517,8 @@ public final class SqliteValueBox: ValueBox {
             self.database.execute("INSERT INTO __meta_tables(name) VALUES (\(table))")
         }
         
+        let startTime = CFAbsoluteTimeGetCurrent()
+        
         var exists = false
         let existsStatement = self.existsStatement(table, key: key)
         if existsStatement.step() {
@@ -488,19 +537,25 @@ public final class SqliteValueBox: ValueBox {
             }
             statement.reset()
         }
+        
+        self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime
     }
     
     public func remove(table: Int32, key: ValueBoxKey) {
         if self.tables.contains(table) {
             
+            let startTime = CFAbsoluteTimeGetCurrent()
+            
             let statement = self.deleteStatement(table, key: key)
             while statement.step() {
             }
             statement.reset()
+            
+            self.writeQueryTime += CFAbsoluteTimeGetCurrent() - startTime
         }
     }
     
-    public func drop() {
+    private func clearStatements() {
         for (_, statement) in self.getStatements {
             statement.destroy()
         }
@@ -565,6 +620,10 @@ public final class SqliteValueBox: ValueBox {
             statement.destroy()
         }
         self.deleteStatements.removeAll()
+    }
+    
+    public func drop() {
+        self.clearStatements()
 
         for table in self.tables {
             self.database.execute("DROP TABLE IF EXISTS t\(table)")

Postbox/ValueBox.swift

@@ -4,6 +4,9 @@ public protocol ValueBox {
     func begin()
     func commit()
     
+    func beginStats()
+    func endStats()
+    
     func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, @noescape values: (ValueBoxKey, ReadBuffer) -> Bool, limit: Int)
     func range(table: Int32, start: ValueBoxKey, end: ValueBoxKey, keys: ValueBoxKey -> Bool, limit: Int)
     func get(table: Int32, key: ValueBoxKey) -> ReadBuffer?

PostboxTests/CodingTests.swift

@@ -165,45 +165,6 @@ class SerializationTests: XCTestCase {
         XCTAssert(decoder.decodeInt32ForKey("a") == 12345, "int32 failed")
     }
     
-    func testIndexBaselinePerformance() {
-        let basePath = "/tmp/postboxtest"
-        do {
-            try NSFileManager.defaultManager().removeItemAtPath(basePath)
-        } catch _ { }
-        let postbox = Postbox<EmptyState>(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: [])
-        postbox._prepareBaselineIndexPerformance()
-        
-        measureBlock {
-            postbox._measureBaselineIndexPerformance()
-        }
-    }
-    
-    func testBlobReadPerformance() {
-        let basePath = "/tmp/postboxtest"
-        do {
-            try NSFileManager.defaultManager().removeItemAtPath(basePath)
-        } catch _ { }
-        let postbox = Postbox<EmptyState>(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: [])
-        postbox._prepareBlobIndexPerformance()
-        
-        measureBlock {
-            postbox._measureBlobReadPerformance()
-        }
-    }
-    
-    func testBlobIndexPerformance() {
-        let basePath = "/tmp/postboxtest"
-        do {
-            try NSFileManager.defaultManager().removeItemAtPath(basePath)
-        } catch _ { }
-        let postbox = Postbox<EmptyState>(basePath: basePath, messageNamespaces: [], absoluteIndexedMessageNamespaces: [])
-        postbox._prepareBlobIndexPerformance()
-        
-        measureBlock {
-            postbox._measureBlobIndexPerformance()
-        }
-    }
-    
     func testKeys() {
         let key1 = ValueBoxKey(length: 8)
         let key2 = ValueBoxKey(length: 8)
@@ -230,7 +191,7 @@ class SerializationTests: XCTestCase {
     }
     
     func testKeyValue() {
-        let basePath = "/tmp/postboxtest"
+        /*let basePath = "/tmp/postboxtest"
         do {
             try NSFileManager.defaultManager().removeItemAtPath(basePath)
         } catch _ { }
@@ -256,6 +217,6 @@ class SerializationTests: XCTestCase {
                     return true
                 }, limit: 10)
             }
-        }
+        }*/
     }
 }

PostboxTests/MessageHistoryIndexTableTests.swift

@@ -0,0 +1,427 @@
+import Foundation
+
+import UIKit
+import XCTest
+
+import Postbox
+@testable import Postbox
+
+private let peerId = PeerId(namespace: 1, id: 1)
+private let namespace: Int32 = 1
+
+private enum Item: Equatable, CustomStringConvertible {
+    case Message(Int32, Int32)
+    case Hole(Int32, Int32, Int32)
+    
+    init(_ item: HistoryIndexEntry) {
+        switch item {
+            case let .Message(index):
+                self = .Message(index.id.id, index.timestamp)
+            case let .Hole(hole):
+                self = .Hole(hole.min, hole.maxIndex.id.id, hole.maxIndex.timestamp)
+        }
+    }
+    
+    var description: String {
+        switch self {
+            case let .Message(id, timestamp):
+                return "Message(\(id), \(timestamp))"
+            case let .Hole(minId, maxId, maxTimestamp):
+                return "Hole(\(minId), \(maxId), \(maxTimestamp))"
+        }
+    }
+}
+
+private func ==(lhs: Item, rhs: Item) -> Bool {
+    switch lhs {
+        case let .Message(id, timestamp):
+            switch rhs {
+                case let .Message(rId, rTimestamp):
+                    return id == rId && timestamp == rTimestamp
+                case .Hole:
+                    return false
+            }
+        case let .Hole(minId, maxId, maxTimestamp):
+            switch rhs {
+                case .Message:
+                    return false
+                case let .Hole(rMinId, rMaxId, rMaxTimestamp):
+                    return minId == rMinId && maxId == rMaxId && maxTimestamp == rMaxTimestamp
+            }
+    }
+}
+
+@testable import Postbox
+
+class MessageHistoryIndexTableTests: XCTestCase {
+    var valueBox: ValueBox?
+    var path: String?
+    
+    var indexTable: MessageHistoryIndexTable?
+    
+    override func setUp() {
+        super.setUp()
+        
+        var randomId: Int64 = 0
+        arc4random_buf(&randomId, 8)
+        path = NSTemporaryDirectory().stringByAppendingString("\(randomId)")
+        self.valueBox = SqliteValueBox(basePath: path!)
+        
+        self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1)
+    }
+    
+    override func tearDown() {
+        super.tearDown()
+        
+        self.indexTable = nil
+        
+        self.valueBox = nil
+        let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!)
+        self.path = nil
+    }
+    
+    func addHole(id: Int32) {
+        self.indexTable!.addHole(MessageId(peerId: peerId, namespace: namespace, id: id))
+    }
+    
+    func addMessage(id: Int32, _ timestamp: Int32) {
+        self.indexTable!.addMessage(MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp))
+    }
+    
+    func fillHole(id: Int32, _ fillType: HoleFillType, _ messages: [(Int32, Int32)]) {
+        self.indexTable!.fillHole(MessageId(peerId: peerId, namespace: namespace, id: id), fillType: fillType, indices: messages.map({MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: $0.0), timestamp: $0.1)}))
+    }
+    
+    func removeMessage(id: Int32) {
+        self.indexTable!.removeMessage(MessageId(peerId: peerId, namespace: namespace, id: id))
+    }
+    
+    private func expect(items: [Item]) {
+        let actualItems = self.indexTable!.debugList(peerId, namespace: namespace).map { return Item($0) }
+        if items != actualItems {
+            XCTFail("Expected\n\(items)\nGot\n\(actualItems)")
+        }
+    }
+    
+    func testEmpty() {
+        expect([])
+    }
+    
+    func testAddMessageToEmpty() {
+        addMessage(100, 100)
+        expect([.Message(100, 100)])
+        
+        addMessage(110, 110)
+        expect([.Message(100, 100), .Message(110, 110)])
+        
+        addMessage(90, 90)
+        expect([.Message(90, 90), .Message(100, 100), .Message(110, 110)])
+    }
+    
+    func testAddHoleToEmpty() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+    }
+    
+    func testAddHoleToFullHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        addHole(110)
+        expect([.Hole(1, Int32.max, Int32.max)])
+    }
+    
+    func testAddMessageToFullHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        addMessage(90, 90)
+        expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)])
+    }
+    
+    func testAddMessageDividingUpperHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        addMessage(90, 90)
+        expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)])
+        addMessage(100, 100)
+        expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, 99, 100), .Message(100, 100), .Hole(101, Int32.max, Int32.max)])
+    }
+    
+    func testAddMessageDividingLowerHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        addMessage(90, 90)
+        expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)])
+        addMessage(80, 80)
+        expect([.Hole(1, 79, 80), .Message(80, 80), .Hole(81, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)])
+    }
+    
+    func testAddMessageOffsettingUpperHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        
+        addMessage(90, 90)
+        expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)])
+        addMessage(91, 91)
+        expect([.Hole(1, 89, 90), .Message(90, 90), .Message(91, 91), .Hole(92, Int32.max, Int32.max)])
+    }
+    
+    func testAddMessageOffsettingLowerHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        
+        addMessage(90, 90)
+        expect([.Hole(1, 89, 90), .Message(90, 90), .Hole(91, Int32.max, Int32.max)])
+        addMessage(89, 89)
+        expect([.Hole(1, 88, 89), .Message(89, 89), .Message(90, 90), .Hole(91, Int32.max, Int32.max)])
+    }
+    
+    func testAddMessageOffsettingLeftmostHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        
+        addMessage(1, 1)
+        
+        expect([.Message(1, 1), .Hole(2, Int32.max, Int32.max)])
+    }
+    
+    func testAddMessageRemovingLefmostHole() {
+        addHole(100)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        
+        addMessage(2, 2)
+        expect([.Hole(1, 1, 2), .Message(2, 2), .Hole(3, Int32.max, Int32.max)])
+        
+        addMessage(1, 1)
+        expect([.Message(1, 1), .Message(2, 2), .Hole(3, Int32.max, Int32.max)])
+    }
+    
+    func testAddHoleLowerThanMessage() {
+        addMessage(100, 100)
+        addHole(1)
+        
+        expect([.Hole(1, 99, 100), .Message(100, 100)])
+    }
+    
+    func testAddHoleHigherThanMessage() {
+        addMessage(100, 100)
+        addHole(200)
+        
+        expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)])
+    }
+    
+    func testIgnoreHigherHole() {
+        addHole(200)
+        expect([.Hole(1, Int32.max, Int32.max)])
+        addHole(400)
+        expect([.Hole(1, Int32.max, Int32.max)])
+    }
+    
+    func testIgnoreHigherHoleAfterMessage() {
+        addMessage(100, 100)
+        addHole(200)
+        expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)])
+        addHole(400)
+        expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)])
+    }
+    
+    func testAddHoleBetweenMessages() {
+        addMessage(100, 100)
+        addMessage(200, 200)
+        addHole(150)
+        
+        expect([.Message(100, 100), .Hole(101, 199, 200), .Message(200, 200)])
+    }
+    
+    func testFillHoleEmpty() {
+        fillHole(1, .Complete, [])
+        expect([])
+    }
+
+    func testFillHoleComplete() {
+        addHole(100)
+        
+        fillHole(1, .Complete, [(100, 100), (200, 200)])
+        expect([.Message(100, 100), .Message(200, 200)])
+    }
+    
+    func testFillHoleUpperToLowerPartial() {
+        addHole(100)
+        
+        fillHole(1, .UpperToLower, [(100, 100), (200, 200)])
+        expect([.Hole(1, 99, 100), .Message(100, 100), .Message(200, 200)])
+    }
+    
+    func testFillHoleUpperToLowerToBounds() {
+        addHole(100)
+        
+        fillHole(1, .UpperToLower, [(1, 1), (200, 200)])
+        expect([.Message(1, 1), .Message(200, 200)])
+    }
+    
+    func testFillHoleLowerToUpperToBounds() {
+        addHole(100)
+        
+        fillHole(1, .LowerToUpper, [(100, 100), (Int32.max, 200)])
+        expect([.Message(100, 100), .Message(Int32.max, 200)])
+    }
+    
+    func testFillHoleLowerToUpperPartial() {
+        addHole(100)
+        
+        fillHole(1, .LowerToUpper, [(100, 100), (200, 200)])
+        expect([.Message(100, 100), .Message(200, 200), .Hole(201, Int32.max, Int32.max)])
+    }
+    
+    func testFillHoleBetweenMessagesUpperToLower() {
+        addHole(1)
+        
+        addMessage(100, 100)
+        addMessage(200, 200)
+        
+        fillHole(199, .UpperToLower, [(150, 150)])
+        
+        expect([.Hole(1, 99, 100), .Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)])
+    }
+    
+    func testFillHoleBetweenMessagesLowerToUpper() {
+        addHole(1)
+        
+        addMessage(100, 100)
+        addMessage(200, 200)
+        
+        fillHole(199, .LowerToUpper, [(150, 150)])
+        
+        expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Hole(151, 199, 200), .Message(200, 200), .Hole(201, Int32.max, Int32.max)])
+    }
+    
+    func testFillHoleBetweenMessagesComplete() {
+        addHole(1)
+        
+        addMessage(100, 100)
+        addMessage(200, 200)
+        
+        fillHole(199, .Complete, [(150, 150)])
+        
+        expect([.Hole(1, 99, 100), .Message(100, 100), .Message(150, 150), .Message(200, 200), .Hole(201, Int32.max, Int32.max)])
+    }
+    
+    func testFillHoleBetweenMessagesWithMessage() {
+        addMessage(200, 200)
+        addMessage(202, 202)
+        addHole(201)
+        addMessage(201, 201)
+        
+        expect([.Message(200, 200), .Message(201, 201), .Message(202, 202)])
+    }
+    
+    func testFillHoleWithNoMessagesComplete() {
+        addMessage(100, 100)
+        addHole(1)
+        
+        fillHole(99, .Complete, [])
+        
+        expect([.Message(100, 100)])
+    }
+    
+    func testFillHoleIgnoreOverMessage() {
+        addMessage(100, 100)
+        addMessage(101, 101)
+        
+        fillHole(100, .Complete, [(90, 90)])
+        
+        expect([.Message(90, 90), .Message(100, 100), .Message(101, 101)])
+    }
+    
+    func testFillHoleWithOverflow() {
+        addMessage(100, 100)
+        addMessage(200, 200)
+        addHole(150)
+        
+        fillHole(199, .UpperToLower, [(150, 150), (300, 300)])
+        
+        expect([.Message(100, 100), .Hole(101, 149, 150), .Message(150, 150), .Message(200, 200), .Message(300, 300)])
+    }
+    
+    func testIgnoreHoleOverMessageBetweenMessages() {
+        addMessage(199, 199)
+        addMessage(200, 200)
+        addHole(200)
+        
+        expect([.Message(199, 199), .Message(200, 200)])
+    }
+    
+    func testMergeHoleAfterDeletingMessage() {
+        addMessage(100, 100)
+        addHole(1)
+        addHole(200)
+        
+        expect([.Hole(1, 99, 100), .Message(100, 100), .Hole(101, Int32.max, Int32.max)])
+        
+        removeMessage(100)
+        
+        expect([.Hole(1, Int32.max, Int32.max)])
+    }
+    
+    func testMergeHoleLowerAfterDeletingMessage() {
+        addMessage(100, 100)
+        addHole(1)
+        addMessage(200, 200)
+        
+        removeMessage(100)
+        
+        expect([.Hole(1, 199, 200), .Message(200, 200)])
+    }
+    
+    func testMergeHoleUpperAfterDeletingMessage() {
+        addMessage(100, 100)
+        addMessage(200, 200)
+        addHole(300)
+        
+        removeMessage(200)
+        
+        expect([.Message(100, 100), .Hole(101, Int32.max, Int32.max)])
+    }
+    
+    func testExtendLowerHoleAfterDeletingMessage() {
+        addMessage(100, 100)
+        addHole(100)
+        
+        removeMessage(100)
+        
+        expect([.Hole(1, Int32.max, Int32.max)])
+    }
+    
+    func testExtendUpperHoleAfterDeletingMessage() {
+        addMessage(100, 100)
+        addHole(101)
+        
+        removeMessage(100)
+        
+        expect([.Hole(1, Int32.max, Int32.max)])
+    }
+    
+    func testDeleteMessageBelowMessage() {
+        addMessage(100, 100)
+        addMessage(200, 200)
+        removeMessage(100)
+        
+        expect([.Message(200, 200)])
+    }
+    
+    func testDeleteMessageAboveMessage() {
+        addMessage(100, 100)
+        addMessage(200, 200)
+        removeMessage(200)
+        
+        expect([.Message(100, 100)])
+    }
+    
+    func testDeleteMessageBetweenMessages() {
+        addMessage(100, 100)
+        addMessage(200, 200)
+        addMessage(300, 300)
+        removeMessage(200)
+        
+        expect([.Message(100, 100), .Message(300, 300)])
+    }
+}

PostboxTests/MessageHistoryTableTests.swift

@@ -0,0 +1,306 @@
+import Foundation
+
+import UIKit
+import XCTest
+
+import Postbox
+@testable import Postbox
+
+private let peerId = PeerId(namespace: 1, id: 1)
+private let namespace: Int32 = 1
+
+private func ==(lhs: (Int32, Int32, String, [Media]), rhs: (Int32, Int32, String, [Media])) -> Bool {
+    if lhs.3.count != rhs.3.count {
+        return false
+    }
+    for i in 0 ..< lhs.3.count {
+        if !lhs.3[i].isEqual(rhs.3[i]) {
+            return false
+        }
+    }
+    return lhs.0 == rhs.0 && lhs.1 == rhs.1 && lhs.2 == rhs.2
+}
+
+private class TestEmbeddedMedia: Media, CustomStringConvertible {
+    var id: MediaId? { return nil }
+    let data: String
+    
+    init(data: String) {
+        self.data = data
+    }
+    
+    required init(decoder: Decoder) {
+        self.data = decoder.decodeStringForKey("s")
+    }
+    
+    func encode(encoder: Encoder) {
+        encoder.encodeString(self.data, forKey: "s")
+    }
+    
+    func isEqual(other: Media) -> Bool {
+        if let other = other as? TestEmbeddedMedia {
+            return self.data == other.data
+        }
+        return false
+    }
+    
+    var description: String {
+        return "TestEmbeddedMedia(\(self.data))"
+    }
+}
+
+private class TestExternalMedia: Media {
+    let id: MediaId?
+    let data: String
+    
+    init(id: Int64, data: String) {
+        self.id = MediaId(namespace: namespace, id: id)
+        self.data = data
+    }
+    
+    required init(decoder: Decoder) {
+        self.id = MediaId(namespace: decoder.decodeInt32ForKey("i.n"), id: decoder.decodeInt64ForKey("i.i"))
+        self.data = decoder.decodeStringForKey("s")
+    }
+    
+    func encode(encoder: Encoder) {
+        encoder.encodeInt32(self.id!.namespace, forKey: "i.n")
+        encoder.encodeInt64(self.id!.id, forKey: "i.i")
+        encoder.encodeString(self.data, forKey: "s")
+    }
+    
+    func isEqual(other: Media) -> Bool {
+        if let other = other as? TestExternalMedia {
+            return self.id == other.id && self.data == other.data
+        }
+        return false
+    }
+    
+    var description: String {
+        return "TestExternalMedia(\(self.id!.id), \(self.data))"
+    }
+}
+
+private enum MediaEntry: Equatable {
+    case Direct(Media, Int)
+    case MessageReference(Int32)
+    
+    init(_ entry: DebugMediaEntry) {
+        switch entry {
+            case let .Direct(media, referenceCount):
+                self = .Direct(media, referenceCount)
+            case let .MessageReference(index):
+                self = MessageReference(index.id.id)
+        }
+    }
+}
+
+private func ==(lhs: MediaEntry, rhs: MediaEntry) -> Bool {
+    switch lhs {
+        case let .Direct(lhsMedia, lhsReferenceCount):
+            switch rhs {
+                case let .Direct(rhsMedia, rhsReferenceCount):
+                    return lhsMedia.isEqual(rhsMedia) && lhsReferenceCount == rhsReferenceCount
+                case .MessageReference:
+                    return false
+            }
+        case let .MessageReference(lhsId):
+            switch rhs {
+                case .Direct:
+                    return false
+                case let .MessageReference(rhsId):
+                    return lhsId == rhsId
+            }
+    }
+}
+
+class MessageHistoryTableTests: XCTestCase {
+    var valueBox: ValueBox?
+    var path: String?
+    
+    var indexTable: MessageHistoryIndexTable?
+    var mediaTable: MessageMediaTable?
+    var mediaCleanupTable: MediaCleanupTable?
+    var historyTable: MessageHistoryTable?
+    
+    override class func setUp() {
+        super.setUp()
+        
+        declareEncodable(TestEmbeddedMedia.self, f: {TestEmbeddedMedia(decoder: $0)})
+        declareEncodable(TestExternalMedia.self, f: {TestExternalMedia(decoder: $0)})
+    }
+    
+    override func setUp() {
+        super.setUp()
+        
+        var randomId: Int64 = 0
+        arc4random_buf(&randomId, 8)
+        path = NSTemporaryDirectory().stringByAppendingString("\(randomId)")
+        self.valueBox = SqliteValueBox(basePath: path!)
+        
+        self.indexTable = MessageHistoryIndexTable(valueBox: self.valueBox!, tableId: 1)
+        self.mediaCleanupTable = MediaCleanupTable(valueBox: self.valueBox!, tableId: 3)
+        self.mediaTable = MessageMediaTable(valueBox: self.valueBox!, tableId: 2, mediaCleanupTable: self.mediaCleanupTable!)
+        self.historyTable = MessageHistoryTable(valueBox: self.valueBox!, tableId: 4, messageHistoryIndexTable: self.indexTable!, messageMediaTable: self.mediaTable!)
+    }
+    
+    override func tearDown() {
+        super.tearDown()
+        
+        self.historyTable = nil
+        self.indexTable = nil
+        self.mediaTable = nil
+        self.mediaCleanupTable = nil
+        
+        self.valueBox = nil
+        let _ = try? NSFileManager.defaultManager().removeItemAtPath(path!)
+        self.path = nil
+    }
+    
+    private func addMessage(id: Int32, _ timestamp: Int32, _ text: String, _ media: [Media] = []) {
+        self.historyTable!.addMessages([StoreMessage(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp, text: text, attributes: [], media: media)])
+    }
+    
+    private func removeMessages(ids: [Int32]) {
+        self.historyTable!.removeMessages(ids.map({ MessageId(peerId: peerId, namespace: namespace, id: $0) }))
+    }
+    
+    func expectMessages(messages: [(Int32, Int32, String, [Media])]) {
+        let actualMessages = self.historyTable!.debugList(peerId).map({ ($0.id.id, $0.timestamp, $0.text, $0.media) })
+        var equal = true
+        if messages.count != actualMessages.count {
+            equal = false
+        } else {
+            for i in 0 ..< messages.count {
+                if !(messages[i] == actualMessages[i]) {
+                    equal = false
+                    break
+                }
+            }
+        }
+        
+        if !equal {
+            XCTFail("Expected\n\(messages)\nActual\n\(actualMessages)")
+        }
+    }
+    
+    private func expectMedia(media: [MediaEntry]) {
+        let actualMedia = self.mediaTable!.debugList().map({MediaEntry($0)})
+        if media != actualMedia {
+            XCTFail("Expected\n\(media)\nActual\n\(actualMedia)")
+        }
+    }
+    
+    private func expectCleanupMedia(media: [Media]) {
+        let actualMedia = self.mediaCleanupTable!.debugList()
+        var equal = true
+        if media.count != actualMedia.count {
+            equal = false
+        } else {
+            for i in 0 ..< media.count {
+                if !media[i].isEqual(actualMedia[i]) {
+                    equal = false
+                    break
+                }
+            }
+        }
+        
+        if !equal {
+            XCTFail("Expected\n\(media)\nActual\n\(actualMedia)")
+        }
+    }
+    
+    func testInsertMessageIntoEmpty() {
+        addMessage(100, 100, "t100")
+        addMessage(200, 200, "t200")
+        
+        expectMessages([(100, 100, "t100", []), (200, 200, "t200", [])])
+    }
+    
+    func testInsertMessageIgnoreOverwrite() {
+        addMessage(100, 100, "t100")
+        addMessage(100, 200, "t200")
+        
+        expectMessages([(100, 100, "t100", [])])
+    }
+    
+    func testInsertMessageWithEmbeddedMedia() {
+        addMessage(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")])
+        
+        expectMessages([(100, 100, "t100", [TestEmbeddedMedia(data: "abc1")])])
+        expectMedia([])
+    }
+    
+    func testInsertMessageWithExternalMedia() {
+        let media = TestExternalMedia(id: 10, data: "abc1")
+        addMessage(100, 100, "t100", [media])
+        
+        expectMessages([(100, 100, "t100", [media])])
+        expectMedia([.MessageReference(100)])
+    }
+    
+    func testUnembedExternalMedia() {
+        let media = TestExternalMedia(id: 10, data: "abc1")
+        addMessage(100, 100, "t100", [media])
+        addMessage(200, 200, "t200", [media])
+        
+        expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])])
+        expectMedia([.Direct(media, 2)])
+    }
+    
+    func testIgnoreOverrideExternalMedia() {
+        let media = TestExternalMedia(id: 10, data: "abc1")
+        let media1 = TestExternalMedia(id: 10, data: "abc2")
+        addMessage(100, 100, "t100", [media])
+        addMessage(200, 200, "t200", [media1])
+        
+        expectMessages([(100, 100, "t100", [media]), (200, 200, "t200", [media])])
+        expectMedia([.Direct(media, 2)])
+    }
+    
+    func testRemoveSingleMessage() {
+        addMessage(100, 100, "t100", [])
+        
+        removeMessages([100])
+        
+        expectMessages([])
+        expectMedia([])
+    }
+    
+    func testRemoveMessageWithEmbeddedMedia() {
+        let media = TestEmbeddedMedia(data: "abc1")
+        addMessage(100, 100, "t100", [media])
+        self.removeMessages([100])
+        
+        expectMessages([])
+        expectMedia([])
+        expectCleanupMedia([media])
+    }
+    
+    func testRemoveOnlyReferenceToExternalMedia() {
+        let media = TestExternalMedia(id: 10, data: "abc1")
+        addMessage(100, 100, "t100", [media])
+        removeMessages([100])
+        
+        expectMessages([])
+        expectMedia([])
+        expectCleanupMedia([media])
+    }
+    
+    func testRemoveReferenceToExternalMedia() {
+        let media = TestExternalMedia(id: 10, data: "abc1")
+        addMessage(100, 100, "t100", [media])
+        addMessage(200, 200, "t200", [media])
+        removeMessages([100])
+        
+        expectMessages([(200, 200, "t200", [media])])
+        expectMedia([.Direct(media, 1)])
+        expectCleanupMedia([])
+        
+        removeMessages([200])
+        
+        expectMessages([])
+        expectMedia([])
+        expectCleanupMedia([media])
+    }
+}