From f564c6ad0158dfbd8606db34bd23cddaed5768aa Mon Sep 17 00:00:00 2001 From: Peter <> Date: Thu, 21 Mar 2019 01:48:13 +0400 Subject: [PATCH] Upgrade global message ids index table --- Postbox.xcodeproj/project.pbxproj | 6 ++ Postbox/Message.swift | 1 + Postbox/Postbox.swift | 2 +- Postbox/PostboxUpgrade_17to18.swift | 2 - Postbox/PostboxUpgrade_18to19.swift | 118 ++++++++++++++++++++++++++++ Postbox/Upgrades.swift | 1 + 6 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 Postbox/PostboxUpgrade_18to19.swift diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index e68a273b72..fa4c3e8f9a 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -62,6 +62,8 @@ D03121001DA579A0006A2A60 /* PeerNotificationSettingsTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */; }; D03229EE1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */; }; D03229EF1E6B33FD0000AF9C /* SqliteInterface.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */; }; + D03387522242E32B007A2CE4 /* PostboxUpgrade_18to19.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03387512242E32A007A2CE4 /* PostboxUpgrade_18to19.swift */; }; + D03387532242E32B007A2CE4 /* PostboxUpgrade_18to19.swift in Sources */ = {isa = PBXBuildFile; fileRef = D03387512242E32A007A2CE4 /* PostboxUpgrade_18to19.swift */; }; D033A6F71C73D512006A2EAB /* MessageHistoryUnsentTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */; }; D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */; }; D037178B20D923CA004773C8 /* CachedItemView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D037178A20D923CA004773C8 /* CachedItemView.swift */; }; @@ -431,6 +433,7 @@ D03120FD1DA562E9006A2A60 /* CachedPeerDataTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CachedPeerDataTable.swift; sourceTree = ""; }; D03120FF1DA579A0006A2A60 /* PeerNotificationSettingsTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = PeerNotificationSettingsTable.swift; sourceTree = ""; }; D03229ED1E6B33FD0000AF9C /* SqliteInterface.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteInterface.swift; sourceTree = ""; }; + D03387512242E32A007A2CE4 /* PostboxUpgrade_18to19.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PostboxUpgrade_18to19.swift; sourceTree = ""; }; D033A6F61C73D512006A2EAB /* MessageHistoryUnsentTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryUnsentTable.swift; sourceTree = ""; }; D033A6F81C73E440006A2EAB /* UnsentMessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = UnsentMessageHistoryView.swift; sourceTree = ""; }; D037178A20D923CA004773C8 /* CachedItemView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CachedItemView.swift; sourceTree = ""; }; @@ -723,6 +726,7 @@ D048B33C203C838500038D05 /* PostboxUpgrade_15to16.swift */, D001388520BD942B007C9721 /* PostboxUpgrade_16to17.swift */, D0119CAC20C9E7A100895300 /* PostboxUpgrade_17to18.swift */, + D03387512242E32A007A2CE4 /* PostboxUpgrade_18to19.swift */, ); name = Upgrade; sourceTree = ""; @@ -1299,6 +1303,7 @@ D0F7B1C01E045C62007EB8A5 /* StringIndexTokens.swift in Sources */, D07047AC1F3DD8D100F6A8D4 /* PendingMessageActionsView.swift in Sources */, D0F7B1CA1E045C6A007EB8A5 /* MessageHistoryUnsentTable.swift in Sources */, + D03387532242E32B007A2CE4 /* PostboxUpgrade_18to19.swift in Sources */, D01BAA561ED1D70C00295217 /* ManagedFile.swift in Sources */, D0BEAF6B1E54B5FB00BD963D /* AccountManagerMetadataTable.swift in Sources */, D0F02CE01E99223E0065DEE2 /* Upgrades.swift in Sources */, @@ -1481,6 +1486,7 @@ D0F02CDF1E99223D0065DEE2 /* Upgrades.swift in Sources */, D08775001E3E3D9F00A97350 /* PreferencesTable.swift in Sources */, D0BEAF6D1E54B77900BD963D /* AccountManagerRecordTable.swift in Sources */, + D03387522242E32B007A2CE4 /* PostboxUpgrade_18to19.swift in Sources */, D03120F81DA53FF4006A2A60 /* PeerPresenceTable.swift in Sources */, D0F9E8731C5A1EE500037222 /* GlobalMessageIdsTable.swift in Sources */, D0AE3EBC1F68261B0069BC90 /* PeerNotificationSettingsView.swift in Sources */, diff --git a/Postbox/Message.swift b/Postbox/Message.swift index 0b7f08f9b1..00b3c659e8 100644 --- a/Postbox/Message.swift +++ b/Postbox/Message.swift @@ -81,6 +81,7 @@ public struct MessageId: Hashable, Comparable, CustomStringConvertible { } } } + public struct MessageIndex: Comparable, Hashable { public let id: MessageId public let timestamp: Int32 diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 10aaab3e79..2d32c75a2a 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -1005,7 +1005,7 @@ public func openPostbox(basePath: String, seedConfiguration: SeedConfiguration) let metadataTable = MetadataTable(valueBox: valueBox, table: MetadataTable.tableSpec(0)) let userVersion: Int32? = metadataTable.userVersion() - let currentUserVersion: Int32 = 18 + let currentUserVersion: Int32 = 19 if let userVersion = userVersion { if userVersion != currentUserVersion { diff --git a/Postbox/PostboxUpgrade_17to18.swift b/Postbox/PostboxUpgrade_17to18.swift index 60f904e0b4..b9a6adfd77 100644 --- a/Postbox/PostboxUpgrade_17to18.swift +++ b/Postbox/PostboxUpgrade_17to18.swift @@ -4,13 +4,11 @@ private func convertNamespaces(value: ReadBuffer, buffer: WriteBuffer) { var count: Int32 = 0 value.read(&count, offset: 0, length: 4) buffer.write(&count, offset: 0, length: 4) - var stateByNamespace: [MessageId.Namespace: PeerReadState] = [:] for _ in 0 ..< count { var namespaceId: Int32 = 0 value.read(&namespaceId, offset: 0, length: 4) buffer.write(&namespaceId, offset: 0, length: 4) - let state: PeerReadState var kind: Int8 = 0 value.read(&kind, offset: 0, length: 1) buffer.write(&kind, offset: 0, length: 1) diff --git a/Postbox/PostboxUpgrade_18to19.swift b/Postbox/PostboxUpgrade_18to19.swift new file mode 100644 index 0000000000..fe8b9ad3bf --- /dev/null +++ b/Postbox/PostboxUpgrade_18to19.swift @@ -0,0 +1,118 @@ +import Foundation + +private struct PrivatePeerId: Hashable { + typealias Namespace = Int32 + typealias Id = Int32 + + let namespace: Namespace + let id: Id + + init(namespace: Namespace, id: Id) { + self.namespace = namespace + self.id = id + } + + init(_ n: Int64) { + self.namespace = Int32((n >> 32) & 0x7fffffff) + self.id = Int32(bitPattern: UInt32(n & 0xffffffff)) + } + + func toInt64() -> Int64 { + return (Int64(self.namespace) << 32) | Int64(bitPattern: UInt64(UInt32(bitPattern: self.id))) + } +} + +func postboxUpgrade_18to19(metadataTable: MetadataTable, valueBox: ValueBox) { + let startTime = CFAbsoluteTimeGetCurrent() + + let globalMessageIdsTable = ValueBoxTable(id: 3, keyType: .int64) + let messageHistoryIndexTable = ValueBoxTable(id: 4, keyType: .binary) + + valueBox.dropTable(globalMessageIdsTable) + + /*var debugPeerIds = Set() + var debugMessageIds = Set() + + valueBox.scan(messageHistoryIndexTable, keys: { key in + let peerId = PrivatePeerId(key.getInt64(0)) + debugPeerIds.insert(peerId) + if peerId.namespace == 0 || peerId.namespace == 1 { + if key.getInt32(8) == 0 { + debugMessageIds.insert(MessageId(peerId: PeerId(peerId.toInt64()), namespace: 0, id: key.getInt32(8 + 4))) + } + } + return true + }) + + var checkPeerIds = Set() + var checkMessageIds = Set()*/ + + let absoluteLowerBound = ValueBoxKey(length: 8) + absoluteLowerBound.setInt64(0, value: 0) + + let absoluteUpperBound = ValueBoxKey(length: 8) + absoluteUpperBound.setInt64(0, value: Int64.max) + + let sharedGlobalIdsKey = ValueBoxKey(length: 8) + let sharedGlobalIdsBuffer = WriteBuffer() + + var totalMessageCount = 0 + + let currentLowerBound: ValueBoxKey = absoluteLowerBound + while true { + var currentPeerId: PrivatePeerId? + valueBox.range(messageHistoryIndexTable, start: currentLowerBound, end: absoluteUpperBound, keys: { + key in + currentPeerId = PrivatePeerId(key.getInt64(0)) + return true + }, limit: 1) + if let currentPeerId = currentPeerId { + /*assert(!checkPeerIds.contains(currentPeerId)) + checkPeerIds.insert(currentPeerId)*/ + + if currentPeerId.namespace == 0 || currentPeerId.namespace == 1 { // CloudUser || CloudGroup + let peerCloudLowerBound = ValueBoxKey(length: 8 + 4) + peerCloudLowerBound.setInt64(0, value: currentPeerId.toInt64()) + peerCloudLowerBound.setInt32(8, value: 0) // Cloud + + let sharedIdPeerId = currentPeerId.toInt64() + + valueBox.range(messageHistoryIndexTable, start: peerCloudLowerBound, end: peerCloudLowerBound.successor, keys: { key in + //assert(key.getInt64(0) == currentPeerId.toInt64()) + //assert(key.getInt32(8) == 0) + + totalMessageCount += 1 + + let id = key.getInt32(8 + 4) + + /*let res = checkMessageIds.insert(MessageId(peerId: PeerId(currentPeerId.toInt64()), namespace: 0, id: id)) + assert(res.inserted)*/ + + sharedGlobalIdsKey.setInt64(0, value: Int64(id)) + + sharedGlobalIdsBuffer.reset() + var idPeerId: Int64 = sharedIdPeerId + var idNamespace: Int32 = 0 + sharedGlobalIdsBuffer.write(&idPeerId, offset: 0, length: 8) + sharedGlobalIdsBuffer.write(&idNamespace, offset: 0, length: 4) + + valueBox.set(globalMessageIdsTable, key: sharedGlobalIdsKey, value: sharedGlobalIdsBuffer) + return true + }, limit: 0) + } + currentLowerBound.setInt64(0, value: currentPeerId.toInt64() + 1) + currentLowerBound.setInt32(8, value: 0) + currentLowerBound.setInt32(8 + 4, value: 0) + } else { + break + } + } + + /*assert(debugPeerIds == checkPeerIds) + assert(debugMessageIds == checkMessageIds)*/ + + let endTime = CFAbsoluteTimeGetCurrent() + postboxLog("Upgrade 18->19 (\(totalMessageCount) messages) took \(endTime - startTime) s") + + metadataTable.setUserVersion(19) +} diff --git a/Postbox/Upgrades.swift b/Postbox/Upgrades.swift index 00c38c391d..b56b7ab171 100644 --- a/Postbox/Upgrades.swift +++ b/Postbox/Upgrades.swift @@ -12,5 +12,6 @@ func registeredUpgrades() -> [Int32: PostboxUpgradeOperation] { dict[15] = .inplace(postboxUpgrade_15to16) dict[16] = .inplace(postboxUpgrade_16to17) dict[17] = .inplace(postboxUpgrade_17to18) + dict[18] = .inplace(postboxUpgrade_18to19) return dict }