diff --git a/Postbox.xcodeproj/project.pbxproj b/Postbox.xcodeproj/project.pbxproj index 2ac23d89f8..826a081033 100644 --- a/Postbox.xcodeproj/project.pbxproj +++ b/Postbox.xcodeproj/project.pbxproj @@ -8,6 +8,7 @@ /* Begin PBXBuildFile section */ D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */; }; + D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0079F591D592E8B00A27A2C /* ContactTable.swift */; }; D00E0FB81B85D192002E4EB5 /* lmdb.h in Headers */ = {isa = PBXBuildFile; fileRef = D00E0FB41B85D192002E4EB5 /* lmdb.h */; }; D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB51B85D192002E4EB5 /* mdb.c */; settings = {COMPILER_FLAGS = "-Wno-conversion -Wno-unreachable-code -Wno-conditional-uninitialized -Wno-format-extra-args -Wno-macro-redefined"; }; }; D00E0FBA1B85D192002E4EB5 /* midl.c in Sources */ = {isa = PBXBuildFile; fileRef = D00E0FB61B85D192002E4EB5 /* midl.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; @@ -43,6 +44,8 @@ D0977F9C1B822DB4009994B2 /* ValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9B1B822DB4009994B2 /* ValueBox.swift */; }; D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */; }; D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */; }; + D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */; }; + D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */; }; D0A7D9451C556CFE0016A115 /* MessageHistoryIndexTableTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */; }; D0C07F6A1B67DB4800966E43 /* SwiftSignalKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */; }; D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */; }; @@ -51,6 +54,8 @@ D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */; }; D0C9DA391C65782500855278 /* SimpleSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0C9DA381C65782500855278 /* SimpleSet.swift */; }; D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */; }; + D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */; }; + D0D949F51D35353900740E02 /* MappedFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0D949F41D35353900740E02 /* MappedFile.swift */; }; D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */ = {isa = PBXBuildFile; fileRef = D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */; }; D0E3A7501B28A7E300A402D9 /* Postbox.h in Headers */ = {isa = PBXBuildFile; fileRef = D0E3A74F1B28A7E300A402D9 /* Postbox.h */; settings = {ATTRIBUTES = (Public, ); }; }; D0E3A7561B28A7E300A402D9 /* Postbox.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D0E3A74A1B28A7E300A402D9 /* Postbox.framework */; }; @@ -86,6 +91,7 @@ /* Begin PBXFileReference section */ D003E4E51B38DBDB00C22CBC /* MessageHistoryView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryView.swift; sourceTree = ""; }; + D0079F591D592E8B00A27A2C /* ContactTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ContactTable.swift; sourceTree = ""; }; D00E0FB41B85D192002E4EB5 /* lmdb.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = lmdb.h; path = submodules/lmdb/libraries/liblmdb/lmdb.h; sourceTree = SOURCE_ROOT; }; D00E0FB51B85D192002E4EB5 /* mdb.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = mdb.c; path = submodules/lmdb/libraries/liblmdb/mdb.c; sourceTree = SOURCE_ROOT; }; D00E0FB61B85D192002E4EB5 /* midl.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = midl.c; path = submodules/lmdb/libraries/liblmdb/midl.c; sourceTree = SOURCE_ROOT; }; @@ -122,6 +128,8 @@ D0977F9B1B822DB4009994B2 /* ValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBox.swift; sourceTree = ""; }; D0977F9D1B8234DF009994B2 /* ValueBoxKey.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ValueBoxKey.swift; sourceTree = ""; }; D0977F9F1B8244D7009994B2 /* SqliteValueBox.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SqliteValueBox.swift; sourceTree = ""; }; + D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessMediaResourceContext.swift; sourceTree = ""; }; + D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OrderStatisticTable.swift; sourceTree = ""; }; D0A7D9441C556CFE0016A115 /* MessageHistoryIndexTableTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryIndexTableTests.swift; sourceTree = ""; }; D0C07F691B67DB4800966E43 /* SwiftSignalKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; path = SwiftSignalKit.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryReadStateTable.swift; sourceTree = ""; }; @@ -130,6 +138,8 @@ D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageHistoryViewTests.swift; sourceTree = ""; }; D0C9DA381C65782500855278 /* SimpleSet.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SimpleSet.swift; sourceTree = ""; }; D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MediaResource.swift; sourceTree = ""; }; + D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = RandomAccessResourceTests.swift; sourceTree = ""; }; + D0D949F41D35353900740E02 /* MappedFile.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MappedFile.swift; sourceTree = ""; }; D0E1DE141C5E1C6900C7826E /* ViewTracker.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ViewTracker.swift; sourceTree = ""; }; D0E3A74A1B28A7E300A402D9 /* Postbox.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Postbox.framework; sourceTree = BUILT_PRODUCTS_DIR; }; D0E3A74E1B28A7E300A402D9 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -194,6 +204,7 @@ D05F09A51C9E9F9300BB6F96 /* MediaResourceStatus.swift */, D0CE63F51CA1CCB2002BC462 /* MediaResource.swift */, D0FC1A3B1CA3FB310056AE9A /* MediaResourceData.swift */, + D09ADF091D2E89F300C8208D /* RandomAccessMediaResourceContext.swift */, ); name = "Media Box"; sourceTree = ""; @@ -254,8 +265,10 @@ D0F9E86E1C5A0E7600037222 /* KeychainTable.swift */, D0F9E8701C5A0E9B00037222 /* PeerTable.swift */, D0C735271C864DF300BB3149 /* PeerChatStateTable.swift */, + D0079F591D592E8B00A27A2C /* ContactTable.swift */, D0C674C71CBB11C600183765 /* MessageHistoryReadStateTable.swift */, D01F7D9A1CBEC390008765C9 /* MessageHistoryInvalidatedReadStateTable.swift */, + D09ADF0B1D2EB83500C8208D /* OrderStatisticTable.swift */, ); name = Tables; sourceTree = ""; @@ -280,6 +293,7 @@ D0F9E8741C5A334100037222 /* SimpleDictionary.swift */, D0C9DA381C65782500855278 /* SimpleSet.swift */, D08CEFB31D2AD8BE0015D3BC /* RedBlackTree.swift */, + D0D949F41D35353900740E02 /* MappedFile.swift */, ); name = Utils; sourceTree = ""; @@ -367,6 +381,7 @@ D0C8FCB61C5C2D200028C27F /* MessageHistoryViewTests.swift */, D060B77A1CF4845A0050BE9B /* ReadStateTableTests.swift */, D02EB8061D2B07F300D07ED3 /* OrderStatisticTreeTests.swift */, + D0D949F21D35302600740E02 /* RandomAccessResourceTests.swift */, ); path = PostboxTests; sourceTree = ""; @@ -501,6 +516,7 @@ D0E3A7821B28ADD000A402D9 /* Postbox.swift in Sources */, D0E3A79E1B28B50400A402D9 /* Message.swift in Sources */, D0C674C81CBB11C600183765 /* MessageHistoryReadStateTable.swift in Sources */, + D09ADF0A1D2E89F300C8208D /* RandomAccessMediaResourceContext.swift in Sources */, D08CEFB41D2AD8BE0015D3BC /* RedBlackTree.swift in Sources */, D044CA2E1C618373002160FF /* ChatListHole.swift in Sources */, D044E1631B2AD677001EE087 /* MurMurHash32.m in Sources */, @@ -512,6 +528,7 @@ D0E1DE151C5E1C6900C7826E /* ViewTracker.swift in Sources */, D0F9E8751C5A334100037222 /* SimpleDictionary.swift in Sources */, D08C713E1C512EA500779C0F /* MessageHistoryTable.swift in Sources */, + D0079F5A1D592E8B00A27A2C /* ContactTable.swift in Sources */, D0F9E8671C58D08900037222 /* ChatListIndexTable.swift in Sources */, D0E3A7A21B28B7DC00A402D9 /* Media.swift in Sources */, D0E3A7881B28AE9C00A402D9 /* Coding.swift in Sources */, @@ -519,8 +536,10 @@ D0977F9E1B8234DF009994B2 /* ValueBoxKey.swift in Sources */, D033A6F91C73E440006A2EAB /* UnsentMessageHistoryView.swift in Sources */, D00E0FB91B85D192002E4EB5 /* mdb.c in Sources */, + D09ADF0C1D2EB83500C8208D /* OrderStatisticTable.swift in Sources */, D0CE63F61CA1CCB2002BC462 /* MediaResource.swift in Sources */, D0F9E86B1C59719800037222 /* ChatListView.swift in Sources */, + D0D949F51D35353900740E02 /* MappedFile.swift in Sources */, D003E4E61B38DBDB00C22CBC /* MessageHistoryView.swift in Sources */, D07516791B2EC90400AE42E0 /* SQLite-Bridging.m in Sources */, D0977FA01B8244D7009994B2 /* SqliteValueBox.swift in Sources */, @@ -551,6 +570,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + D0D949F31D35302600740E02 /* RandomAccessResourceTests.swift in Sources */, D044E15E1B2ACB9C001EE087 /* CodingTests.swift in Sources */, D0F9E8611C57766A00037222 /* MessageHistoryTableTests.swift in Sources */, D0C8FCB71C5C2D200028C27F /* MessageHistoryViewTests.swift in Sources */, @@ -640,6 +660,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; }; name = Hockeyapp; }; @@ -655,6 +676,7 @@ LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 3.0; }; name = Hockeyapp; }; @@ -777,6 +799,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -806,6 +829,7 @@ PROVISIONING_PROFILE_SPECIFIER = X834Q8SBVP/; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; }; name = Release; }; @@ -826,6 +850,7 @@ LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 3.0; }; name = Debug; }; @@ -842,6 +867,7 @@ PRODUCT_BUNDLE_IDENTIFIER = "org.telegram.$(PRODUCT_NAME:rfc1034identifier)"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; + SWIFT_VERSION = 3.0; }; name = Release; }; diff --git a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist index dd754c89ca..9eb93eab1b 100644 --- a/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist +++ b/Postbox.xcodeproj/xcuserdata/peter.xcuserdatad/xcschemes/xcschememanagement.plist @@ -12,7 +12,7 @@ PostboxTests.xcscheme orderHint - 3 + 4 SuppressBuildableAutocreation diff --git a/Postbox/ChatListView.swift b/Postbox/ChatListView.swift index 7efc992bd3..f3e0bc64e7 100644 --- a/Postbox/ChatListView.swift +++ b/Postbox/ChatListView.swift @@ -280,7 +280,7 @@ final class MutableChatListView { } addedEntries += self.entries - addedEntries.sort(isOrderedBefore: { $0.index < $1.index }) + addedEntries.sort(by: { $0.index < $1.index }) var i = addedEntries.count - 1 while i >= 1 { if addedEntries[i].index.id == addedEntries[i - 1].index.id { diff --git a/Postbox/Coding.swift b/Postbox/Coding.swift index 08b32c12a4..c03cfacd1c 100644 --- a/Postbox/Coding.swift +++ b/Postbox/Coding.swift @@ -98,7 +98,7 @@ public func ==(lhs: MemoryBuffer, rhs: MemoryBuffer) -> Bool { } public final class WriteBuffer: MemoryBuffer { - var offset = 0 + public var offset = 0 public override init() { super.init(memory: malloc(32), capacity: 32, length: 0, freeWhenDone: true) @@ -155,7 +155,7 @@ public final class WriteBuffer: MemoryBuffer { } public final class ReadBuffer: MemoryBuffer { - var offset = 0 + public var offset = 0 public init(memory: UnsafeMutablePointer, length: Int, freeWhenDone: Bool) { super.init(memory: memory, capacity: length, length: length, freeWhenDone: freeWhenDone) @@ -165,24 +165,24 @@ public final class ReadBuffer: MemoryBuffer { super.init(memory: memoryBufferNoCopy.memory, capacity: memoryBufferNoCopy.length, length: memoryBufferNoCopy.length, freeWhenDone: false) } - func dataNoCopy() -> Data { + public func dataNoCopy() -> Data { return Data(bytesNoCopy: UnsafeMutablePointer(self.memory), count: self.length, deallocator: .none) } - func read(_ data: UnsafeMutablePointer, offset: Int, length: Int) { + public func read(_ data: UnsafeMutablePointer, offset: Int, length: Int) { memcpy(data + offset, self.memory + self.offset, length) self.offset += length } - func skip(_ length: Int) { + public func skip(_ length: Int) { self.offset += length } - func reset() { + public func reset() { self.offset = 0 } - func sharedBufferNoCopy() -> ReadBuffer { + public func sharedBufferNoCopy() -> ReadBuffer { return ReadBuffer(memory: memory, length: length, freeWhenDone: false) } } diff --git a/Postbox/ContactTable.swift b/Postbox/ContactTable.swift new file mode 100644 index 0000000000..2045f4d47d --- /dev/null +++ b/Postbox/ContactTable.swift @@ -0,0 +1,9 @@ +import Foundation + +final class ContactTable: Table { + + + override func beforeCommit() { + + } +} diff --git a/Postbox/LmdbValueBox.swift b/Postbox/LmdbValueBox.swift index d20c12264b..67263c1c05 100644 --- a/Postbox/LmdbValueBox.swift +++ b/Postbox/LmdbValueBox.swift @@ -113,10 +113,10 @@ public final class LmdbValueBox: ValueBox { var createDirectory = false var isDirectory: ObjCBool = false as ObjCBool - if FileManager.default().fileExists(atPath: path, isDirectory: &isDirectory) { - if !isDirectory { + if FileManager.default.fileExists(atPath: path, isDirectory: &isDirectory) { + if !isDirectory.boolValue { do { - try FileManager.default().removeItem(atPath: path) + try FileManager.default.removeItem(atPath: path) } catch _ { } createDirectory = true } @@ -127,7 +127,7 @@ public final class LmdbValueBox: ValueBox { if createDirectory { do { - try FileManager.default().createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) + try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true, attributes: nil) } catch _ { } } diff --git a/Postbox/MappedFile.swift b/Postbox/MappedFile.swift new file mode 100644 index 0000000000..8c57238fde --- /dev/null +++ b/Postbox/MappedFile.swift @@ -0,0 +1,51 @@ +import Foundation + +public final class MappedFile { + private var handle: Int32 + private var currentSize: Int + private var memory: UnsafeMutablePointer + + public init(path: String) { + self.handle = open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR) + + var value = stat() + stat(path, &value) + self.currentSize = Int(value.st_size) + + self.memory = mmap(nil, self.currentSize, PROT_READ | PROT_WRITE, MAP_SHARED, self.handle, 0) + } + + deinit { + munmap(self.memory, self.currentSize) + close(self.handle) + } + + public var size: Int { + get { + return self.currentSize + } set(value) { + if value != self.currentSize { + munmap(self.memory, self.currentSize) + ftruncate(self.handle, off_t(value)) + self.currentSize = value + self.memory = mmap(nil, self.currentSize, PROT_READ | PROT_WRITE, MAP_SHARED, self.handle, 0) + } + } + } + + public func synchronize() { + msync(self.memory, self.currentSize, MS_ASYNC) + } + + public func write(at range: Range, from data: UnsafePointer) { + memcpy(self.memory.advanced(by: range.lowerBound), data, range.count) + } + + public func read(at range: Range, to data: UnsafeMutablePointer) { + memcpy(data, self.memory.advanced(by: range.lowerBound), range.count) + } + + public func clear() { + memset(self.memory, 0, self.currentSize) + } +} diff --git a/Postbox/MediaBox.swift b/Postbox/MediaBox.swift index 0a1124a70f..3a310e80db 100644 --- a/Postbox/MediaBox.swift +++ b/Postbox/MediaBox.swift @@ -25,6 +25,12 @@ private func fileSize(_ path: String) -> Int { return Int(value.st_size) } +public enum ResourceDataRangeMode { + case complete + case incremental + case partial +} + public final class MediaBox { let basePath: String let buffer = WriteBuffer() @@ -34,9 +40,10 @@ public final class MediaBox { private var statusContexts: [String: ResourceStatusContext] = [:] private var dataContexts: [String: ResourceDataContext] = [:] + private var randomAccessContexts: [String: RandomAccessMediaResourceContext] = [:] - private var wrappedFetchResource = Promise<(MediaResource, Int) -> Signal>() - public var fetchResource: ((MediaResource, Int) -> Signal)? { + private var wrappedFetchResource = Promise<(MediaResource, Range) -> Signal>() + public var fetchResource: ((MediaResource, Range) -> Signal)? { didSet { if let fetchResource = self.fetchResource { wrappedFetchResource.set(.single(fetchResource)) @@ -47,15 +54,25 @@ public final class MediaBox { } lazy var ensureDirectoryCreated: Void = { - try! FileManager.default().createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) + try! FileManager.default.createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) }() public init(basePath: String) { self.basePath = basePath + + let _ = self.ensureDirectoryCreated + } + + private func fileNameForId(_ id: String) -> String { + return id } private func pathForId(_ id: String) -> String { - return "\(self.basePath)/\(id)" + return "\(self.basePath)/\(fileNameForId(id))" + } + + private func streamingPathForId(_ id: String) -> String { + return "\(self.basePath)/\(id)_stream" } public func resourceStatus(_ resource: MediaResource) -> Signal { @@ -126,7 +143,7 @@ public final class MediaBox { } } - public func resourceData(_ resource: MediaResource, complete: Bool = true) -> Signal { + public func resourceData(_ resource: MediaResource, pathExtension: String? = nil, complete: Bool = true) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -135,7 +152,15 @@ public final class MediaBox { let currentSize = fileSize(path) if currentSize >= resource.size { - subscriber.putNext(MediaResourceData(path: path, size: currentSize)) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !FileManager.default.fileExists(atPath: pathWithExtension) { + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: currentSize)) + } else { + subscriber.putNext(MediaResourceData(path: path, size: currentSize)) + } subscriber.putCompletion() } else { let dataContext: ResourceDataContext @@ -148,25 +173,79 @@ public final class MediaBox { let index: Bag<(MediaResourceData) -> Void>.Index if complete { + var checkedForSymlink = false index = dataContext.completeDataSubscribers.add { data in - subscriber.putNext(data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) + } else { + subscriber.putNext(data) + } + if data.size >= resource.size { subscriber.putCompletion() } } + if dataContext.data.size >= resource.size { - subscriber.putNext(dataContext.data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) + } else { + subscriber.putNext(dataContext.data) + } } else { - subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: 0)) + } else { + subscriber.putNext(MediaResourceData(path: dataContext.data.path, size: 0)) + } } } else { + var checkedForSymlink = false index = dataContext.progresiveDataSubscribers.add { data in - subscriber.putNext(data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: data.size)) + } else { + subscriber.putNext(data) + } if data.size >= resource.size { subscriber.putCompletion() } } - subscriber.putNext(dataContext.data) + if let pathExtension = pathExtension { + let pathWithExtension = path + ".\(pathExtension)" + if !checkedForSymlink && !FileManager.default.fileExists(atPath: pathWithExtension) { + checkedForSymlink = true + let _ = try? FileManager.default.createSymbolicLink(atPath: pathWithExtension, withDestinationPath: self.fileNameForId(resource.id)) + } + + subscriber.putNext(MediaResourceData(path: pathWithExtension, size: dataContext.data.size)) + } else { + subscriber.putNext(dataContext.data) + } } disposable.set(ActionDisposable { @@ -191,7 +270,127 @@ public final class MediaBox { } } - public func fetchedResource(_ resource: MediaResource, interactive: Bool) -> Signal { + private func randomAccessContext(for resource: MediaResource) -> RandomAccessMediaResourceContext { + assert(self.dataQueue.isCurrent()) + + let dataContext: RandomAccessMediaResourceContext + if let current = self.randomAccessContexts[resource.id] { + dataContext = current + } else { + let path = self.pathForId(resource.id) + ".random" + dataContext = RandomAccessMediaResourceContext(path: path, size: resource.size, fetchRange: { [weak self] range in + let disposable = MetaDisposable() + + if let strongSelf = self { + strongSelf.dataQueue.async { + let fetch = strongSelf.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in + return fetch(resource, range) + } + var offset = 0 + disposable.set(fetch.start(next: { [weak strongSelf] data in + if let strongSelf = strongSelf { + strongSelf.dataQueue.async { + if let dataContext = strongSelf.randomAccessContexts[resource.id] { + let storeRange = RandomAccessResourceStoreRange(offset: range.lowerBound + offset, data: data) + offset += data.count + dataContext.storeRanges([storeRange]) + } + } + } + })) + } + } + + return disposable + }) + self.randomAccessContexts[resource.id] = dataContext + } + return dataContext + } + + public func fetchedResourceData(_ resource: MediaResource, in range: Range) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.dataQueue.async { + let dataContext = self.randomAccessContext(for: resource) + + let listener = dataContext.addListenerForFetchedData(in: range) + + disposable.set(ActionDisposable { [weak self] in + if let strongSelf = self { + strongSelf.dataQueue.async { + if let dataContext = strongSelf.randomAccessContexts[resource.id] { + dataContext.removeListenerForFetchedData(listener) + if !dataContext.hasDataListeners() { + let _ = strongSelf.randomAccessContexts.removeValue(forKey: resource.id) + } + } + } + } + }) + } + + return disposable + } + } + + public func resourceData(_ resource: MediaResource, in range: Range, mode: ResourceDataRangeMode = .complete) -> Signal { + return Signal { subscriber in + let disposable = MetaDisposable() + + self.dataQueue.async { + let dataContext = self.randomAccessContext(for: resource) + + let listenerMode: RandomAccessResourceDataRangeMode + switch mode { + case .complete: + listenerMode = .Complete + case .incremental: + listenerMode = .Incremental + case .partial: + listenerMode = .Partial + } + + var offset = 0 + + let listener = dataContext.addListenerForData(in: range, mode: listenerMode, updated: { [weak self] data in + if let strongSelf = self { + strongSelf.dataQueue.async { + subscriber.putNext(data) + + switch mode { + case .complete, .partial: + offset = max(offset, data.count) + case .incremental: + offset += data.count + } + if offset == range.count { + subscriber.putCompletion() + } + } + } + }) + + disposable.set(ActionDisposable { [weak self] in + if let strongSelf = self { + strongSelf.dataQueue.async { + if let dataContext = strongSelf.randomAccessContexts[resource.id] { + dataContext.removeListenerForData(listener) + if !dataContext.hasDataListeners() { + let _ = strongSelf.randomAccessContexts.removeValue(forKey: resource.id) + } + } + } + } + }) + } + + return disposable + } + } + + public func fetchedResource(_ resource: MediaResource) -> Signal { return Signal { subscriber in let disposable = MetaDisposable() @@ -225,8 +424,8 @@ public final class MediaBox { var offset = currentSize var fd: Int32? - dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> mapToSignal { fetch -> Signal in - return fetch(resource, offset) + dataContext.fetchDisposable = (self.wrappedFetchResource.get() |> take(1) |> mapToSignal { fetch -> Signal in + return fetch(resource, currentSize ..< resource.size) } |> afterDisposed { if let fd = fd { close(fd) @@ -250,6 +449,8 @@ public final class MediaBox { offset += data.count let updatedSize = offset + dataContext.data = MediaResourceData(path: path, size: updatedSize) + for subscriber in dataContext.progresiveDataSubscribers.copyItems() { subscriber(MediaResourceData(path: path, size: updatedSize)) } diff --git a/Postbox/MessageHistoryIndexTable.swift b/Postbox/MessageHistoryIndexTable.swift index 3d6c9906e8..44e56fbe3f 100644 --- a/Postbox/MessageHistoryIndexTable.swift +++ b/Postbox/MessageHistoryIndexTable.swift @@ -364,7 +364,7 @@ final class MessageHistoryIndexTable: Table { return true }, limit: 1) - let sortedByIdMessages = messages.sorted(isOrderedBefore: {$0.id < $1.id}) + let sortedByIdMessages = messages.sorted(by: {$0.id < $1.id}) var remainingMessages = sortedByIdMessages diff --git a/Postbox/MessageHistoryReadStateTable.swift b/Postbox/MessageHistoryReadStateTable.swift index a9fe295875..d86aafbeee 100644 --- a/Postbox/MessageHistoryReadStateTable.swift +++ b/Postbox/MessageHistoryReadStateTable.swift @@ -205,7 +205,7 @@ final class MessageHistoryReadStateTable: Table { print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId.id) (before: \(states.namespaces))") } - if state.maxIncomingReadId < messageId.id || messageId.id == topMessageId { + if state.maxIncomingReadId < messageId.id || (messageId.id == topMessageId && state.count != 0) { var (deltaCount, holes) = incomingStatsInRange(state.maxIncomingReadId + 1, messageId.id) if traceReadStates { diff --git a/Postbox/MessageHistoryTagsTable.swift b/Postbox/MessageHistoryTagsTable.swift index 822ec4d25e..55c6412b36 100644 --- a/Postbox/MessageHistoryTagsTable.swift +++ b/Postbox/MessageHistoryTagsTable.swift @@ -38,6 +38,25 @@ class MessageHistoryTagsTable: Table { self.valueBox.remove(self.tableId, key: self.key(tagMask, index: index, key: self.sharedKey)) } + func entryLocation(at index: MessageIndex, tagMask: MessageTags) -> MessageHistoryEntryLocation? { + if let _ = self.valueBox.get(self.tableId, key: self.key(tagMask, index: index)) { + var greaterCount = 0 + self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.upperBound(tagMask, peerId: index.id.peerId), keys: { _ in + greaterCount += 1 + return true + }, limit: 0) + + var lowerCount = 0 + self.valueBox.range(self.tableId, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask, peerId: index.id.peerId), keys: { _ in + lowerCount += 1 + return true + }, limit: 0) + + return MessageHistoryEntryLocation(index: lowerCount, count: greaterCount + lowerCount + 1) + } + return nil + } + func indicesAround(_ tagMask: MessageTags, index: MessageIndex, count: Int) -> (indices: [MessageIndex], lower: MessageIndex?, upper: MessageIndex?) { var lowerEntries: [MessageIndex] = [] var upperEntries: [MessageIndex] = [] diff --git a/Postbox/MessageHistoryView.swift b/Postbox/MessageHistoryView.swift index e48b4e0a1b..05371fe0ae 100644 --- a/Postbox/MessageHistoryView.swift +++ b/Postbox/MessageHistoryView.swift @@ -21,31 +21,67 @@ public func ==(lhs: MessageHistoryViewId, rhs: MessageHistoryViewId) -> Bool { } enum MutableMessageHistoryEntry { - case IntermediateMessageEntry(IntermediateMessage) - case MessageEntry(Message) - case HoleEntry(MessageHistoryHole) + case IntermediateMessageEntry(IntermediateMessage, MessageHistoryEntryLocation?) + case MessageEntry(Message, MessageHistoryEntryLocation?) + case HoleEntry(MessageHistoryHole, MessageHistoryEntryLocation?) var index: MessageIndex { switch self { - case let .IntermediateMessageEntry(message): + case let .IntermediateMessageEntry(message, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .MessageEntry(message): + case let .MessageEntry(message, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .HoleEntry(hole): + case let .HoleEntry(hole, _): return hole.maxIndex } } + + func updatedLocation(_ location: MessageHistoryEntryLocation?) -> MutableMessageHistoryEntry { + switch self { + case let .IntermediateMessageEntry(message, _): + return .IntermediateMessageEntry(message, location) + case let .MessageEntry(message, _): + return .MessageEntry(message, location) + case let .HoleEntry(hole, _): + return .HoleEntry(hole, location) + } + } +} + +public struct MessageHistoryEntryLocation: Equatable { + public let index: Int + public let count: Int + + var predecessor: MessageHistoryEntryLocation? { + if index == 0 { + return nil + } else { + return MessageHistoryEntryLocation(index: index - 1, count: count) + } + } + + var successor: MessageHistoryEntryLocation? { + if index == count - 1 { + return nil + } else { + return MessageHistoryEntryLocation(index: index + 1, count: count) + } + } +} + +public func ==(lhs: MessageHistoryEntryLocation, rhs: MessageHistoryEntryLocation) -> Bool { + return lhs.index == rhs.index && lhs.count == rhs.count } public enum MessageHistoryEntry: Comparable { - case MessageEntry(Message) - case HoleEntry(MessageHistoryHole) + case MessageEntry(Message, MessageHistoryEntryLocation?) + case HoleEntry(MessageHistoryHole, MessageHistoryEntryLocation?) public var index: MessageIndex { switch self { - case let .MessageEntry(message): + case let .MessageEntry(message, _): return MessageIndex(id: message.id, timestamp: message.timestamp) - case let .HoleEntry(hole): + case let .HoleEntry(hole, _): return hole.maxIndex } } @@ -53,20 +89,20 @@ public enum MessageHistoryEntry: Comparable { public func ==(lhs: MessageHistoryEntry, rhs: MessageHistoryEntry) -> Bool { switch lhs { - case let .MessageEntry(lhsMessage): + case let .MessageEntry(lhsMessage, lhsLocation): switch rhs { case .HoleEntry: return false - case let .MessageEntry(rhsMessage): - if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags { + case let .MessageEntry(rhsMessage, rhsLocation): + if MessageIndex(lhsMessage) == MessageIndex(rhsMessage) && lhsMessage.flags == rhsMessage.flags && lhsLocation == rhsLocation { return true } return false } - case let .HoleEntry(lhsHole): + case let .HoleEntry(lhsHole, lhsLocation): switch rhs { - case let .HoleEntry(rhsHole): - return lhsHole == rhsHole + case let .HoleEntry(rhsHole, rhsLocation): + return lhsHole == rhsHole && lhsLocation == rhsLocation case .MessageEntry: return false } @@ -199,13 +235,13 @@ final class MutableMessageHistoryView { switch operation { case let .InsertHole(hole): if tagMask == nil || (hole.tags & unwrappedTagMask) != 0 { - if self.add(.HoleEntry(hole), holeFillDirections: holeFillDirections) { + if self.add(.HoleEntry(hole, nil), holeFillDirections: holeFillDirections) { hasChanges = true } } case let .InsertMessage(intermediateMessage): if tagMask == nil || (intermediateMessage.tags.rawValue & unwrappedTagMask) != 0 { - if self.add(.IntermediateMessageEntry(intermediateMessage), holeFillDirections: holeFillDirections) { + if self.add(.IntermediateMessageEntry(intermediateMessage, nil), holeFillDirections: holeFillDirections) { hasChanges = true } } @@ -218,8 +254,8 @@ final class MutableMessageHistoryView { //self.combinedReadState = combinedReadState case let .UpdateEmbeddedMedia(index, embeddedMediaData): for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(message) = self.entries[i] where MessageIndex(message) == index { - self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia)) + if case let .IntermediateMessageEntry(message, _) = self.entries[i] where MessageIndex(message) == index { + self.entries[i] = .IntermediateMessageEntry(IntermediateMessage(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: embeddedMediaData, referencedMedia: message.referencedMedia), nil) hasChanges = true break } @@ -230,7 +266,7 @@ final class MutableMessageHistoryView { if !updatedMedia.isEmpty { for i in 0 ..< self.entries.count { switch self.entries[i] { - case let .MessageEntry(message): + case let .MessageEntry(message, _): var rebuild = false for media in message.media { if let mediaId = media.id, _ = updatedMedia[mediaId] { @@ -251,10 +287,10 @@ final class MutableMessageHistoryView { } } let updatedMessage = Message(stableId: message.stableId, id: message.id, timestamp: message.timestamp, flags: message.flags, tags: message.tags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: messageMedia, peers: message.peers, associatedMessages: message.associatedMessages) - self.entries[i] = .MessageEntry(updatedMessage) + self.entries[i] = .MessageEntry(updatedMessage, nil) hasChanges = true } - case let .IntermediateMessageEntry(message): + case let .IntermediateMessageEntry(message, _): var rebuild = false for mediaId in message.referencedMedia { if let media = updatedMedia[mediaId] where media?.id != mediaId { @@ -427,16 +463,16 @@ final class MutableMessageHistoryView { } func render(_ renderIntermediateMessage: (IntermediateMessage) -> Message) { - if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage) = earlier { - self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage)) + if let earlier = self.earlier, case let .IntermediateMessageEntry(intermediateMessage, location) = earlier { + self.earlier = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } - if let later = self.later, case let .IntermediateMessageEntry(intermediateMessage) = later { - self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage)) + if let later = self.later, case let .IntermediateMessageEntry(intermediateMessage, location) = later { + self.later = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } for i in 0 ..< self.entries.count { - if case let .IntermediateMessageEntry(intermediateMessage) = self.entries[i] { - self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage)) + if case let .IntermediateMessageEntry(intermediateMessage, location) = self.entries[i] { + self.entries[i] = .MessageEntry(renderIntermediateMessage(intermediateMessage), location) } } } @@ -459,7 +495,7 @@ final class MutableMessageHistoryView { while i >= 0 || j < self.entries.count { if j < self.entries.count { - if case let .HoleEntry(hole) = self.entries[j] { + if case let .HoleEntry(hole, _) = self.entries[j] { if self.anchorIndex.index.id.namespace == hole.id.namespace { if self.anchorIndex.index.id.id >= hole.min && self.anchorIndex.index.id.id <= hole.maxIndex.id.id { return (hole, .AroundIndex(self.anchorIndex.index)) @@ -471,7 +507,7 @@ final class MutableMessageHistoryView { } if i >= 0 { - if case let .HoleEntry(hole) = self.entries[i] { + if case let .HoleEntry(hole, _) = self.entries[i] { if self.anchorIndex.index.id.namespace == hole.id.namespace { if self.anchorIndex.index.id.id >= hole.min && self.anchorIndex.index.id.id <= hole.maxIndex.id.id { return (hole, .AroundIndex(self.anchorIndex.index)) @@ -506,12 +542,12 @@ public final class MessageHistoryView { var entries: [MessageHistoryEntry] = [] for entry in mutableView.entries { switch entry { - case let .HoleEntry(hole): - entries.append(.HoleEntry(hole)) - case let .MessageEntry(message): - entries.append(.MessageEntry(message)) + case let .HoleEntry(hole, location): + entries.append(.HoleEntry(hole, location)) + case let .MessageEntry(message, location): + entries.append(.MessageEntry(message, location)) case .IntermediateMessageEntry: - assertionFailure("got IntermediateMessageEntry") + assertionFailure("unexpected IntermediateMessageEntry in MessageHistoryView.init()") } } self.entries = entries @@ -545,7 +581,7 @@ public final class MessageHistoryView { } if let _ = maxNamespaceIndex where index + 1 < entries.count { for i in index + 1 ..< entries.count { - if case let .MessageEntry(message) = entries[i] where !message.flags.contains(.Incoming) { + if case let .MessageEntry(message, _) = entries[i] where !message.flags.contains(.Incoming) { maxNamespaceIndex = MessageIndex(message) } else { break diff --git a/Postbox/OrderStatisticTable.swift b/Postbox/OrderStatisticTable.swift new file mode 100644 index 0000000000..758abf9f86 --- /dev/null +++ b/Postbox/OrderStatisticTable.swift @@ -0,0 +1,59 @@ +import Foundation + +class MessageOrderStatisticTable: Table { + override init(valueBox: ValueBox, tableId: Int32) { + super.init(valueBox: valueBox, tableId: tableId) + } + + /*private func update(peerId: PeerId, tagMask: MessageTags, id: Int32, count: Int) { + let key = ValueBoxKey(length: 8 + 4 + 4) + key.setInt64(0, value: peerId.toInt64()) + key.setUInt32(8, value: tagMask.rawValue) + + var writeValue: Int32 = 0 + let buffer = MemoryBuffer(memory: &writeValue, capacity: 4, length: 4, freeWhenDone: false) + var idx = id + while (idx <= 1000000) { + key.setInt32(8 + 4, value: idx) + var value: Int32 = 0 + if let data = self.valueBox.get(self.tableId, key: key) { + data.read(&value, offset: 0, length: 4) + } + if value == 0 { + self.valueBox.remove(self.tableId, key: key) + } else { + writeValue = value + self.valueBox.set(self.tableId, key: key, value: buffer) + } + idx += idx & -idx + } + } + + private func get(peerId: PeerId, tagMask: MessageTags, id: Int32) -> Int32 { + let key = ValueBoxKey(length: 8 + 4) + key.setInt64(0, value: peerId.toInt64()) + + var idx = id + + var sum: Int32 = 0 + while (idx > 0) { + key.setInt32(8, value: idx) + var value: Int32 = 0 + if let data = self.valueBox.get(self.tableId, key: key) { + data.read(&value, offset: 0, length: 4) + } + sum += value + idx -= idx & -idx + } + + return sum + } + + func set(_ id: MessageId, count: Int) { + self.update(peerId: id.peerId, id: id.id, count: count) + } + + func get(_ id: MessageId) -> Int32 { + return self.get(peerId: id.peerId, id: id.id) + }*/ +} diff --git a/Postbox/Postbox.swift b/Postbox/Postbox.swift index 3a3a6c3cf5..ab87c588b7 100644 --- a/Postbox/Postbox.swift +++ b/Postbox/Postbox.swift @@ -193,7 +193,9 @@ public final class Postbox { self.globalMessageIdsNamespace = globalMessageIdsNamespace self.seedConfiguration = seedConfiguration - let _ = try? FileManager.default().removeItem(atPath: self.basePath + "/media") + print("MediaBox path: \(self.basePath + "/media")") + + //let _ = try? FileManager.default.removeItem(atPath: self.basePath + "/media") self.mediaBox = MediaBox(basePath: self.basePath + "/media") self.openDatabase() @@ -202,9 +204,9 @@ public final class Postbox { private func debugSaveState(name: String) { self.queue.justDispatch({ let path = self.basePath + name - let _ = try? FileManager.default().removeItem(atPath: path) + let _ = try? FileManager.default.removeItem(atPath: path) do { - try FileManager.default().copyItem(atPath: self.basePath, toPath: path) + try FileManager.default.copyItem(atPath: self.basePath, toPath: path) } catch (let e) { print("(Postbox debugSaveState: error \(e))") } @@ -214,9 +216,9 @@ public final class Postbox { private func debugRestoreState(name: String) { self.queue.justDispatch({ let path = self.basePath + name - let _ = try? FileManager.default().removeItem(atPath: self.basePath) + let _ = try? FileManager.default.removeItem(atPath: self.basePath) do { - try FileManager.default().copyItem(atPath: path, toPath: self.basePath) + try FileManager.default.copyItem(atPath: path, toPath: self.basePath) } catch (let e) { print("(Postbox debugRestoreState: error \(e))") } @@ -228,7 +230,7 @@ public final class Postbox { let startTime = CFAbsoluteTimeGetCurrent() do { - try FileManager.default().createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) + try FileManager.default.createDirectory(atPath: self.basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } @@ -441,9 +443,9 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, nil)) case let .Hole(index): - entries.append(.HoleEntry(index)) + entries.append(.HoleEntry(index, nil)) } } return entries @@ -465,9 +467,9 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, nil)) case let .Hole(index): - entries.append(.HoleEntry(index)) + entries.append(.HoleEntry(index, nil)) } } @@ -475,9 +477,9 @@ public final class Postbox { if let intermediateLower = intermediateLower { switch intermediateLower { case let .Message(message): - lower = .IntermediateMessageEntry(message) + lower = .IntermediateMessageEntry(message, nil) case let .Hole(index): - lower = .HoleEntry(index) + lower = .HoleEntry(index, nil) } } @@ -485,13 +487,17 @@ public final class Postbox { if let intermediateUpper = intermediateUpper { switch intermediateUpper { case let .Message(message): - upper = .IntermediateMessageEntry(message) + upper = .IntermediateMessageEntry(message, nil) case let .Hole(index): - upper = .HoleEntry(index) + upper = .HoleEntry(index, nil) } } - return (entries: entries, lower: lower, upper: upper) + if let tagMask = tagMask { + return addLocationsToMessageHistoryViewEntries(tagMask: tagMask, earlier: lower, later: upper, entries: entries) + } else { + return (entries: entries, lower: lower, upper: upper) + } } private func fetchLaterHistoryEntries(_ peerId: PeerId, index: MessageIndex?, count: Int, tagMask: MessageTags? = nil) -> [MutableMessageHistoryEntry] { @@ -505,9 +511,9 @@ public final class Postbox { for entry in intermediateEntries { switch entry { case let .Message(message): - entries.append(.IntermediateMessageEntry(message)) + entries.append(.IntermediateMessageEntry(message, nil)) case let .Hole(index): - entries.append(.HoleEntry(index)) + entries.append(.HoleEntry(index, nil)) } } return entries @@ -735,6 +741,32 @@ public final class Postbox { return self.aroundMessageHistoryViewForPeerId(peerId, index: index, count: count, anchorIndex: MessageHistoryAnchorIndex(index: anchorIndex, exact: true), unreadIndex: nil, fixedCombinedReadState: fixedCombinedReadState, tagMask: tagMask) } + private func addLocationsToMessageHistoryViewEntries(tagMask: MessageTags, earlier: MutableMessageHistoryEntry?, later: MutableMessageHistoryEntry?, entries: [MutableMessageHistoryEntry]) -> ([MutableMessageHistoryEntry], MutableMessageHistoryEntry?, MutableMessageHistoryEntry?) { + if let firstEntry = entries.first { + if let location = self.messageHistoryTagsTable.entryLocation(at: firstEntry.index, tagMask: tagMask) { + var mappedEarlier = earlier?.updatedLocation(location.predecessor) + var mappedEntries: [MutableMessageHistoryEntry] = [] + var previousLocation: MessageHistoryEntryLocation? + for i in 0 ..< entries.count { + if i == 0 { + mappedEntries.append(entries[i].updatedLocation(location)) + previousLocation = location + } else { + previousLocation = previousLocation?.successor + mappedEntries.append(entries[i].updatedLocation(previousLocation)) + } + } + previousLocation = previousLocation?.successor + var mappedLater = later?.updatedLocation(previousLocation) + return (mappedEntries, mappedEarlier, mappedLater) + } else { + return (entries, earlier, later) + } + } else { + return (entries, earlier, later) + } + } + private func aroundMessageHistoryViewForPeerId(_ peerId: PeerId, index: MessageIndex, count: Int, anchorIndex: MessageHistoryAnchorIndex, unreadIndex: MessageIndex?, fixedCombinedReadState: CombinedPeerReadState?, tagMask: MessageTags? = nil) -> Signal<(MessageHistoryView, ViewUpdateType), NoError> { return Signal { subscriber in let disposable = MetaDisposable() @@ -742,6 +774,7 @@ public final class Postbox { self.queue.justDispatch({ //print("+ queue \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") let (entries, earlier, later) = self.fetchAroundHistoryEntries(index, count: count, tagMask: tagMask) + print("aroundMessageHistoryViewForPeerId fetchAroundHistoryEntries \((CFAbsoluteTimeGetCurrent() - startTime) * 1000.0) ms") let mutableView = MutableMessageHistoryView(id: MessageHistoryViewId(peerId: peerId, id: self.takeNextViewId()), anchorIndex: anchorIndex, combinedReadState: fixedCombinedReadState ?? self.readStateTable.getCombinedState(peerId), earlier: earlier, entries: entries, later: later, tagMask: tagMask, count: count) diff --git a/Postbox/RandomAccessMediaResourceContext.swift b/Postbox/RandomAccessMediaResourceContext.swift new file mode 100644 index 0000000000..08006169c5 --- /dev/null +++ b/Postbox/RandomAccessMediaResourceContext.swift @@ -0,0 +1,469 @@ +import Foundation +import SwiftSignalKit + +public struct RandomAccessResourceStoreRange { + private let offset: Int + private let data: Data + + public init(offset: Int, data: Data) { + self.offset = offset + self.data = data + } +} + +public enum RandomAccessResourceDataRangeMode { + case Complete + case Partial + case Incremental + case None +} + +private final class RandomAccessBlockRangeListener: Hashable { + private let id: Int32 + private let range: Range + private let blockSize: Int + private let blocks: Range + private let mode: RandomAccessResourceDataRangeMode + private let updated: (Data) -> Void + + private var missingBlocks: Set + + init(id: Int32, range: Range, blockSize: Int, blocks: Range, missingBlocks: Set, mode: RandomAccessResourceDataRangeMode, updated: (Data) -> Void) { + self.id = id + self.range = range + self.blockSize = blockSize + self.blocks = blocks + self.mode = mode + self.updated = updated + self.missingBlocks = missingBlocks + } + + var hashValue: Int { + return Int(self.id) + } + + func updateMissingBlocks(addedBlocks: Set, fetchData: @noescape(Range) -> Data) { + if self.missingBlocks.isEmpty { + return + } + + switch self.mode { + case .Complete: + self.missingBlocks.subtract(addedBlocks) + if self.missingBlocks.isEmpty { + self.updated(fetchData(self.range)) + } + case .Incremental, .Partial: + var continuousBlockCount = 0 + for index in CountableRange(self.blocks) { + if self.missingBlocks.contains(index) { + break + } + continuousBlockCount += 1 + } + + self.missingBlocks.subtract(addedBlocks) + + var updatedContinuousBlockCount = 0 + for index in CountableRange(self.blocks) { + if self.missingBlocks.contains(index) { + break + } + updatedContinuousBlockCount += 1 + } + + if updatedContinuousBlockCount > continuousBlockCount { + if self.mode == .Partial { + self.updated(fetchData(self.range)) + } else { + let firstBlock = self.blocks.lowerBound + continuousBlockCount + let lastBlock = self.blocks.lowerBound + updatedContinuousBlockCount + + var startOffset = firstBlock * self.blockSize + if firstBlock == 0 { + startOffset = self.range.lowerBound + } + var endOffset = lastBlock * self.blockSize + if lastBlock == self.blocks.upperBound { + endOffset = self.range.upperBound + } + + self.updated(fetchData(startOffset ..< endOffset)) + } + } + case .None: + break + } + } +} + +private func ==(lhs: RandomAccessBlockRangeListener, rhs: RandomAccessBlockRangeListener) -> Bool { + return lhs.id == rhs.id +} + +private struct FetchRange: Hashable { + let range: Range + + var hashValue: Int { + return self.range.lowerBound ^ self.range.upperBound + } +} + +private func ==(lhs: FetchRange, rhs: FetchRange) -> Bool { + return lhs.range == rhs.range +} + +public final class RandomAccessMediaResourceContext { + private let path: String + private let size: Int + + private var file: MappedFile + private var readyBlocks = Set() + let blockSize: Int + private let fragmentBlockCount: Int + let blockCount: Int + + private var nextBlockRangeListenerId: Int32 = 0 + private var blockRangeListenersByBlockIndex: [Int: [RandomAccessBlockRangeListener]] = [:] + private var blockRangeListenerSet: [Int32: RandomAccessBlockRangeListener] = [:] + private var fetchedBlockRangeListenerSet: [Int32: Range] = [:] + + private var fetchRanges = Set() + private var fetchDisposables: [FetchRange: Disposable] = [:] + + private var fetchRange: (Range) -> Disposable + + public init(path: String, size: Int, fetchRange: (Range) -> Disposable) { + self.path = path + self.size = size + self.fetchRange = fetchRange + + let metadataPath = path + ".meta" + self.file = MappedFile(path: metadataPath) + + self.blockSize = 64 * 1024 + self.fragmentBlockCount = 16 + self.blockCount = size / self.blockSize + (size % self.blockSize == 0 ? 0 : 1) + let expectedSize = 4 + self.blockCount * 1 + + if self.file.size != expectedSize { + self.file.size = expectedSize + var version: Int32 = 1 + self.file.write(at: 0 ..< 4, from: &version) + } + + var version: Int32 = 0 + self.file.read(at: 0 ..< 4, to: &version) + precondition(version == 1) + + for i in 0 ..< blockCount { + var blockStatus: Int8 = 0 + self.file.read(at: (4 + i) ..< (4 + i + 1), to: &blockStatus) + if blockStatus != 0 { + self.readyBlocks.insert(i) + } + } + } + + public func storeRanges(_ ranges: [RandomAccessResourceStoreRange]) { + var blockStatus: Int8 = 1 + + var blocksWithListeners = Set() + + for range in ranges.sorted(by: { $0.offset < $1.offset }) { + var offset = range.offset + let endOffset = offset + range.data.count + assert(offset % self.blockSize == 0) + assert(offset >= 0) + + assert(endOffset == self.size || (endOffset < self.size && endOffset % self.blockSize == 0)) + + var fragmentCache: (Int, MappedFile)? + + var blockIndex = offset / self.blockSize + while offset < endOffset { + let fragmentIndex = blockIndex / self.fragmentBlockCount + + let currentFragmentSize = min(self.size - fragmentIndex * self.fragmentBlockCount * self.blockSize, self.fragmentBlockCount * self.blockSize) + let currentBlockSize = min(self.size - blockIndex * self.blockSize, self.blockSize) + + let fragmentFile: MappedFile + if let fragmentCache = fragmentCache, fragmentCache.0 == fragmentIndex { + fragmentFile = fragmentCache.1 + } else { + fragmentCache?.1.synchronize() + fragmentFile = MappedFile(path: self.path + ".\(fragmentIndex)") + fragmentFile.size = currentFragmentSize + fragmentCache = (fragmentIndex, fragmentFile) + } + + let fragmentBlockIndex = blockIndex % self.fragmentBlockCount + let fragmentBlockOffset = fragmentBlockIndex * self.blockSize + + range.data.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + fragmentFile.write(at: fragmentBlockOffset ..< (fragmentBlockOffset + currentBlockSize), from: bytes.advanced(by: offset - range.offset)) + } + + self.readyBlocks.insert(blockIndex) + if let listeners = self.blockRangeListenersByBlockIndex[blockIndex], !listeners.isEmpty { + blocksWithListeners.insert(blockIndex) + } + + self.file.write(at: (4 + blockIndex) ..< (4 + blockIndex + 1), from: &blockStatus) + + offset += blockSize + blockIndex += 1 + } + + fragmentCache?.1.synchronize() + } + self.file.synchronize() + + var updateListeners = Set() + + for index in blocksWithListeners { + if let listeners = self.blockRangeListenersByBlockIndex.removeValue(forKey: index) { + for listener in listeners { + updateListeners.insert(listener) + } + } + } + + for listener in updateListeners { + listener.updateMissingBlocks(addedBlocks: blocksWithListeners, fetchData: { range in + return self.fetchContiguousReadyData(in: range) + }) + } + } + + public func fetchContiguousReadyData(in dataRange: Range) -> Data { + var data = Data() + + var fragmentCache: (Int, MappedFile)? + + let firstBlock = dataRange.lowerBound / self.blockSize + let lastBlock = dataRange.upperBound / self.blockSize + (dataRange.upperBound % self.blockSize == 0 ? 0 : 1) + + let range = firstBlock ..< lastBlock + + var hadNonReadyBlock = false + for blockIndex in range { + if !self.readyBlocks.contains(blockIndex) { + hadNonReadyBlock = true + break + } + + let fragmentIndex = blockIndex / self.fragmentBlockCount + + let currentFragmentSize = min(self.size - fragmentIndex * self.fragmentBlockCount * self.blockSize, self.fragmentBlockCount * self.blockSize) + let currentBlockSize = min(self.size - blockIndex * self.blockSize, self.blockSize) + + let fragmentFile: MappedFile + if let fragmentCache = fragmentCache, fragmentCache.0 == fragmentIndex { + fragmentFile = fragmentCache.1 + } else { + //fragmentCache?.1.synchronize() + fragmentFile = MappedFile(path: self.path + ".\(fragmentIndex)") + assert(fragmentFile.size == currentFragmentSize) + fragmentCache = (fragmentIndex, fragmentFile) + } + + let fragmentBlockIndex = blockIndex % self.fragmentBlockCount + let fragmentBlockOffset = fragmentBlockIndex * self.blockSize + + var currentBlockStart = 0 + if blockIndex == firstBlock { + currentBlockStart = dataRange.lowerBound % self.blockSize + } + + var currentBlockEnd = currentBlockSize + if blockIndex == lastBlock - 1 && (dataRange.upperBound % self.blockSize) != 0 { + currentBlockEnd = dataRange.upperBound % self.blockSize + } + + data.count += currentBlockEnd - currentBlockStart + data.withUnsafeMutableBytes { (bytes: UnsafeMutablePointer) -> Void in + fragmentFile.read(at: (fragmentBlockOffset + currentBlockStart) ..< (fragmentBlockOffset + currentBlockEnd), to: bytes.advanced(by: data.count - (currentBlockEnd - currentBlockStart))) + } + } + + if !hadNonReadyBlock { + assert(data.count == dataRange.count, "\(data.count) != \(dataRange.count)") + } + + return data + } + + public func missingBlocks(in set: Set) -> Set { + return set.subtracting(self.readyBlocks) + } + + private func missingBlocks(in range: Range) -> Set { + var result = Set() + for index in CountableRange(range) { + if !self.readyBlocks.contains(index) { + result.insert(index) + } + } + return result + } + + public func addListenerForData(in range: Range, mode: RandomAccessResourceDataRangeMode, updated: (Data) -> Void) -> Int32 { + let firstBlock = range.lowerBound / self.blockSize + let lastBlock = range.upperBound / self.blockSize + (range.upperBound % self.blockSize == 0 ? 0 : 1) + + let missingBlocks = self.missingBlocks(in: firstBlock ..< lastBlock) + + if missingBlocks.isEmpty { + updated(self.fetchContiguousReadyData(in: range)) + return -1 + } else { + if missingBlocks.count < (firstBlock ..< lastBlock).count { + switch mode { + case .Complete: + break + case .Incremental, .Partial: + updated(self.fetchContiguousReadyData(in: range)) + case .None: + break + } + } + + let id = self.nextBlockRangeListenerId + self.nextBlockRangeListenerId += 1 + let listener = RandomAccessBlockRangeListener(id: id, range: range, blockSize: self.blockSize, blocks: firstBlock ..< lastBlock, missingBlocks: missingBlocks, mode: mode, updated: { data in + updated(data) + }) + + for index in missingBlocks { + if self.blockRangeListenersByBlockIndex[index] == nil { + self.blockRangeListenersByBlockIndex[index] = [listener] + } else { + self.blockRangeListenersByBlockIndex[index]!.append(listener) + } + } + + self.blockRangeListenerSet[listener.id] = listener + + return listener.id + } + } + + public func removeListenerForData(_ id: Int32) { + if id == -1 { + return + } + + if let listener = self.blockRangeListenerSet.removeValue(forKey: id) { + for index in listener.missingBlocks { + if self.blockRangeListenersByBlockIndex[index] != nil { + if let listenerIndex = self.blockRangeListenersByBlockIndex[index]?.index(where: { $0 === listener }) { + self.blockRangeListenersByBlockIndex[index]?.remove(at: listenerIndex) + if let isEmpty = self.blockRangeListenersByBlockIndex[index]?.isEmpty, isEmpty { + self.blockRangeListenersByBlockIndex.removeValue(forKey: index) + } + } + } + } + } + } + + public func addListenerForFetchedData(in range: Range) -> Int32 { + let firstBlock = range.lowerBound / self.blockSize + let lastBlock = range.upperBound / self.blockSize + (range.upperBound % self.blockSize == 0 ? 0 : 1) + + let missingBlocks = self.missingBlocks(in: firstBlock ..< lastBlock) + + if missingBlocks.isEmpty { + return -1 + } else { + let id = self.nextBlockRangeListenerId + self.nextBlockRangeListenerId += 1 + + self.fetchedBlockRangeListenerSet[id] = firstBlock ..< lastBlock + + self.updateFetchDisposables() + + return id + } + } + + public func removeListenerForFetchedData(_ id: Int32) { + let _ = self.fetchedBlockRangeListenerSet.removeValue(forKey: id) + + self.updateFetchDisposables() + } + + public func hasDataListeners() -> Bool { + return !self.blockRangeListenerSet.isEmpty || !self.fetchedBlockRangeListenerSet.isEmpty + } + + private func updateFetchDisposables() { + var fetchRangeList: [Range] = [] + + for listener in self.fetchedBlockRangeListenerSet.values.sorted(by: { $0.lowerBound < $1.lowerBound }) { + if !fetchRangeList.isEmpty { + if fetchRangeList[fetchRangeList.count - 1].upperBound >= listener.lowerBound { + let upperBound = max(fetchRangeList[fetchRangeList.count - 1].upperBound, listener.upperBound) + fetchRangeList[fetchRangeList.count - 1] = fetchRangeList[fetchRangeList.count - 1].lowerBound ..< upperBound + } else { + fetchRangeList.append(listener) + } + } else { + fetchRangeList.append(listener) + } + } + + for listener in self.fetchedBlockRangeListenerSet.values { + for i in 0 ..< fetchRangeList.count { + if fetchRangeList[i].contains(listener.lowerBound) { + if fetchRangeList[i].lowerBound < listener.lowerBound { + fetchRangeList.insert(listener.lowerBound ..< fetchRangeList[i].upperBound, at: i + 1) + fetchRangeList[i] = fetchRangeList[i].lowerBound ..< listener.lowerBound + } + break + } + } + } + + let blockRanges = Set(fetchRangeList.map({ FetchRange(range: $0) })) + + let removedRanges = self.fetchRanges.subtracting(blockRanges) + let addedRanges = blockRanges.subtracting(self.fetchRanges) + + for blockRange in removedRanges { + self.fetchDisposables.removeValue(forKey: blockRange)?.dispose() + } + + for blockRange in addedRanges { + let disposables = DisposableSet() + + let blocksToFetch = self.missingBlocks(in: blockRange.range) + + var contiguousBlockRanges: [Range] = [] + + for blockIndex in blocksToFetch.sorted() { + if !contiguousBlockRanges.isEmpty { + if contiguousBlockRanges[contiguousBlockRanges.count - 1].upperBound == blockIndex { + contiguousBlockRanges[contiguousBlockRanges.count - 1] = contiguousBlockRanges[contiguousBlockRanges.count - 1].lowerBound ..< (blockIndex + 1) + } else { + contiguousBlockRanges.append(blockIndex ..< (blockIndex + 1)) + } + } else { + contiguousBlockRanges.append(blockIndex ..< (blockIndex + 1)) + } + } + + for blockRange in contiguousBlockRanges { + let lowerBoundOffset = blockRange.lowerBound * self.blockSize + let upperBoundOffset = min(self.size, blockRange.upperBound * self.blockSize) + + disposables.add(self.fetchRange(lowerBoundOffset ..< upperBoundOffset)) + } + + self.fetchDisposables[blockRange] = disposables + } + + self.fetchRanges = blockRanges + } +} diff --git a/Postbox/RedBlackTree.swift b/Postbox/RedBlackTree.swift index c711a6d3ff..e69de29bb2 100644 --- a/Postbox/RedBlackTree.swift +++ b/Postbox/RedBlackTree.swift @@ -1,448 +0,0 @@ -public class RBTNode { - private(set) public var color: Bool - private(set) public var left: RBTNode! - private(set) public var right: RBTNode! - private(set) public var parent: RBTNode! - private(set) public var data: Int - init() { - self.data = -1 - self.color = true - self.left = nil - self.right = nil - self.parent = nil - } - init(rootData: Int) { - self.data = rootData - self.color = true //0 is black 1 is red - self.left = nil - self.right = nil - self.parent = nil - } - deinit { - //print("Node: \(self.data) is bein deinitialized") - } - - public func grandparent() -> RBTNode? { - if self.parent === nil || self.parent.parent === nil { - return nil - } else { - return self.parent.parent - } - } - public func sibling() -> RBTNode? { - if self.parent === nil || self.parent.right === nil || self.parent.left === nil { - return nil - } - if self === self.parent!.left! { - return self.parent!.right! - } else { - return self.parent!.left! - } - } -} -public class RBTree { - private(set) public var root: RBTNode? - init(rootData: Int) { - root = RBTNode(rootData: rootData) - root!.color = false - } - init() { - root = nil - } - public func depth() -> Int { - let n = depth(root!) - return n - } - //return the max depth of the tree - private func depth(_ rooty: RBTNode?) -> Int { - if rooty == nil { - return 0 - } else { - return 1+(max(depth(root!.left), depth(root!.right))) - } - } - - public func inOrder() { - inOrder(root) - } - //Prints the in order traversal of the current tree - private func inOrder(_ root: RBTNode?) { - if self.root == nil { - //print("The tree is empty.") - } - if root == nil { - return - } - inOrder(root!.left) - //print("Data: \(root!.data) Color: \(root!.color)") - inOrder(root!.right) - } - /* - Basic Algorithm: - Let Q be P's right child. - Set P's right child to be Q's left child. - [Set Q's left-child's parent to P] - Set Q's left child to be P. - [Set P's parent to Q] - */ - private func leftRotate(_ x: RBTNode) { - let newRoot = x.right! - x.right = newRoot.left - if newRoot.left !== nil { - newRoot.left.parent = x - } - newRoot.parent = x.parent - if x.parent === nil { - root = newRoot - } else if x === x.parent.left { - x.parent.left = newRoot - } else { - x.parent.right = newRoot - } - newRoot.left = x - x.parent = newRoot - } - /* - Basic Algorithm: - Let P be Q's left child. - Set Q's left child to be P's right child. - [Set P's right-child's parent to Q] - Set P's right child to be Q. - [Set Q's parent to P] - */ - private func rightRotate(_ x: RBTNode) { - let newRoot = x.left! - x.left = newRoot.right - if newRoot.right !== nil { - newRoot.right.parent = x - } - newRoot.parent = x.parent - if x.parent === nil { - root = newRoot - } else if x === x.parent.right { - x.parent.right = newRoot - } else { - x.parent.left = newRoot - } - newRoot.right = x - x.parent = newRoot - } - public func insertFixup(_ value: Int) { - let inserted = find(value) - //print("Inserted Node: \(inserted!.data)") - insertCase1(inserted) - } - //Case where root is the only node - private func insertCase1(_ inserted: RBTNode?) { - let myroot = self.root! - if myroot === inserted! { - self.root!.color = false - } - insertCase2(inserted!) - } - //Case for inserting a node as a child of a black node - private func insertCase2(_ inserted: RBTNode?) { - if inserted!.parent!.color == false { - return - } - insertCase3(inserted!) - } - //Insert case for if the parent is black and parent's siblng is black - private func insertCase3(_ inserted: RBTNode?) { - if inserted!.parent!.sibling() != nil && - inserted!.parent!.sibling()!.color == true { - inserted!.parent!.color = false - inserted!.parent!.sibling()!.color = false - let g = inserted!.grandparent - g()!.color = true - if g()!.parent == nil { - g()!.color = false - } - } - insertCase4(inserted) - } - //Insert case for Node N is left of parent and parent is right of grandparent - private func insertCase4(_ insert: RBTNode?) { - var inserted = insert - if (inserted! === inserted!.parent!.right) && - (inserted!.grandparent()!.left === inserted!.parent!) { - - leftRotate(inserted!.parent) - inserted! = inserted!.left - } else if (inserted! === inserted!.parent!.left) && - (inserted!.grandparent()!.right === inserted!.parent!) { - - rightRotate(inserted!.parent) - inserted! = inserted!.right - } - insertCase5(inserted) - } - //Insert case for Node n where parent is red and parent's sibling is black - private func insertCase5(_ inserted: RBTNode?) { - if inserted!.parent!.color == true && - (inserted!.parent!.sibling()?.color == false || - inserted!.parent!.sibling() == nil) { - - if inserted! === inserted!.parent!.left && inserted!.grandparent()!.left === inserted!.parent! { - inserted!.parent.color = false - inserted!.grandparent()?.color = true - if inserted! === inserted!.parent!.left { - rightRotate(inserted!.grandparent()!) - } - } else if inserted! === inserted!.parent!.right && inserted!.grandparent()!.right === inserted!.parent! { - inserted!.parent.color = false - inserted!.grandparent()?.color = true - leftRotate(inserted!.grandparent()!) - } - } - } - - public func insert(_ value: Int) { - insert(value, parent: root!) - insertFixup(value) - } - //Basic BST insert implementation - private func insert(_ value: Int, parent: RBTNode) { - if self.root == nil { - self.root = RBTNode(rootData: value) - return - } else if value < parent.data { - if let left = parent.left { - insert(value, parent: left) - } else { - parent.left = RBTNode(rootData: value) - parent.left?.parent = parent - } - } else { - if let right = parent.right { - insert(value, parent: right) - } else { - parent.right = RBTNode(rootData: value) - parent.right?.parent = parent - } - } - } - public func find(_ data: Int) -> RBTNode? { - return find(root!, data: data) - } - //Returns the reference to the RBTNode whos data was requested - private func find(_ root: RBTNode, data: Int) -> RBTNode? { - if data == root.data { - return root - } - if root.data != data && root.right == nil && root.left == nil { - return nil - } else if data > root.data { - return find(root.right, data: data) - } else if data < root.data { - return find(root.left, data: data) - } else { - return nil - } - } - - //DELETION HELPER FUNCTIONS - public func remove(_ value: Int) { - let toRemove = find(value) - if toRemove == nil { - return - } - } - //Transplant the positions of two nodes in the RBTree - public func replaceNode(_ n1: RBTNode, n2: RBTNode) { - let temp = n1.data - let temp_color = n1.color - n1.data = n2.data - n1.color = n2.color - n2.data = temp - n2.color = temp_color - } - //returns the node with the minimum value in the subtree - public func minimum(_ node: RBTNode) -> RBTNode { - var minimumNode = node - while minimumNode.left !== nil { - minimumNode = minimumNode.left - } - return minimumNode - } - //Returns the next largest node in the tree - public func successor(_ node: RBTNode) -> RBTNode? { - var nextLargestNode = node - if nextLargestNode.right !== nil { - return minimum(nextLargestNode.right) - } - var successor = nextLargestNode.parent - while successor !== nil && nextLargestNode === successor!.right { - nextLargestNode = successor! - successor = successor!.parent - } - return successor - } - //Returns the next smallest node in the tree - public func predecessor(_ node: RBTNode) -> RBTNode { - var nextSmallestNode = node - if nextSmallestNode.left !== nil { - return minimum(nextSmallestNode.left) - } - var successor = nextSmallestNode.parent - while successor !== nil && nextSmallestNode === successor!.left { - nextSmallestNode = successor! - successor = successor!.parent - } - return successor! - } - //Returns the node with the largest value in the subtree - public func maximum(_ rootnode: RBTNode) -> RBTNode { - var rootNode = rootnode - while rootNode.right !== nil { - rootNode = rootNode.right - } - return rootNode - } - //call to remove a node from the tree - public func delete(_ x: Int) { - let toDel = find(x) - deleteNode(toDel!) - } - //Function call for removing a node - private func deleteNode(_ todel: RBTNode?) { - var toDel = todel - //case for if todel is the only node in the tree - if toDel!.left === nil && toDel!.right === nil && toDel!.parent === nil { - toDel = nil - self.root = nil - return - } - //case for if toDell is a red node w/ no children - if toDel!.left === nil && toDel!.right === nil && toDel!.color == true { - if toDel!.parent.left === toDel! { - toDel!.parent.left = nil - toDel = nil - } else if toDel!.parent === nil { - toDel = nil - } else { - toDel?.parent.right = nil - toDel = nil - } - return - } - //case for toDel having two children - if toDel!.left !== nil && toDel!.right !== nil { - let pred = maximum(toDel!.left!) - toDel!.data = pred.data - toDel! = pred - } - //case for toDel having one child - var child: RBTNode? = nil - if toDel!.right === nil { - child = toDel!.left - } else { - child = toDel!.right - } - if toDel!.color == false && child !== nil { - toDel!.color = child!.color - deleteCase1(toDel!) - } - - if child !== nil { - replaceNode(toDel!, n2: child!) - if toDel!.parent === nil && child !== nil { - child!.color = false - } - } - if toDel!.parent.left === toDel! { - toDel!.parent.left = nil - toDel = nil - return - } else if toDel!.parent === nil { - toDel = nil - return - } else { - toDel?.parent.right = nil - toDel = nil - return - } - } - //delete case for if parent is nil after deletion - private func deleteCase1(_ toDel: RBTNode?) { - if toDel?.parent === nil { - return - } else { - deleteCase2(toDel) - } - } - //case to fix tree after deletion and sibling is red - private func deleteCase2(_ toDel: RBTNode?) { - let sibling = toDel!.sibling() - if sibling?.color == true { - toDel!.parent.color = true - sibling?.color = false - if toDel! === toDel!.parent.left { - leftRotate(toDel!.parent) - } else { - rightRotate(toDel!.parent) - } - deleteCase3(toDel!) - } - } - //delete case for fixing tree when parnet is black and sibling is black and sibling.children are also black - private func deleteCase3(_ toDel: RBTNode?) { - if toDel!.parent?.color == false && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left!.color == false && - toDel!.sibling()?.right!.color == false { - - toDel!.sibling()?.color = true - toDel!.parent?.color = false - } else { - deleteCase4(toDel) - } - } - private func deleteCase4(_ toDel: RBTNode?) { - if toDel!.parent?.color == true && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left!.color == false && - toDel!.sibling()?.right!.color == false { - - toDel!.sibling()?.color = true - toDel!.parent.color = false - } else { - deleteCase5(toDel) - } - } - //delete case for fixing tree if toDel is a left child and sibling(n) is black and children of sibling(n) are black and white respectibely - private func deleteCase5(_ toDel: RBTNode?) { - if toDel! === toDel!.parent?.left && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left.color == true && - toDel!.sibling()?.right.color == false { - - toDel!.sibling()?.color = true - toDel!.sibling()?.left?.color = false - rightRotate(toDel!.sibling()!) - } - //opposite case - else if toDel! === toDel!.parent?.right && - toDel!.sibling()?.color == false && - toDel!.sibling()?.left.color == false && - toDel!.sibling()?.right.color == true { - - toDel!.sibling()?.color = true - toDel!.sibling()?.right?.color = false - leftRotate(toDel!.sibling()!) - } - } - //final rotations to be done after deleting a black node from the tree - private func deleteCase6(_ toDel: RBTNode?) { - let color = toDel!.sibling()?.color - toDel!.parent?.color = color! - toDel!.parent?.color = false - if toDel! === toDel!.parent.left { - toDel!.sibling()?.right?.color = false - leftRotate(toDel!.parent!) - } else { - toDel!.sibling()?.left?.color = false - rightRotate(toDel!.parent!) - } - } -} diff --git a/Postbox/SqliteValueBox.swift b/Postbox/SqliteValueBox.swift index 8d5d64fbc7..cf61a25bd1 100644 --- a/Postbox/SqliteValueBox.swift +++ b/Postbox/SqliteValueBox.swift @@ -62,7 +62,7 @@ private struct SqlitePreparedStatement { } public final class SqliteValueBox: ValueBox { - private let lock = RecursiveLock() + private let lock = NSRecursiveLock() private let basePath: String private var database: Database! @@ -102,7 +102,7 @@ public final class SqliteValueBox: ValueBox { lock.lock() do { - try FileManager.default().createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) + try FileManager.default.createDirectory(atPath: basePath, withIntermediateDirectories: true, attributes: nil) } catch _ { } let path = basePath + "/db_sqlite" let database = Database(path) @@ -697,7 +697,7 @@ public final class SqliteValueBox: ValueBox { self.database = nil self.lock.unlock() - let _ = try? FileManager.default().removeItem(atPath: self.basePath) + let _ = try? FileManager.default.removeItem(atPath: self.basePath) self.database = self.openDatabase() tables.removeAll() diff --git a/PostboxTests/ChatListTableTests.swift b/PostboxTests/ChatListTableTests.swift index 6caf32aefb..3178b94934 100644 --- a/PostboxTests/ChatListTableTests.swift +++ b/PostboxTests/ChatListTableTests.swift @@ -115,7 +115,7 @@ class ChatListTableTests: XCTestCase { self.chatListTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } diff --git a/PostboxTests/MessageHistoryIndexTableTests.swift b/PostboxTests/MessageHistoryIndexTableTests.swift index 43c44dde75..1875a1e6f9 100644 --- a/PostboxTests/MessageHistoryIndexTableTests.swift +++ b/PostboxTests/MessageHistoryIndexTableTests.swift @@ -88,7 +88,7 @@ class MessageHistoryIndexTableTests: XCTestCase { self.indexTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } diff --git a/PostboxTests/MessageHistoryTableTests.swift b/PostboxTests/MessageHistoryTableTests.swift index 713bd60ff2..0dc11db243 100644 --- a/PostboxTests/MessageHistoryTableTests.swift +++ b/PostboxTests/MessageHistoryTableTests.swift @@ -248,7 +248,7 @@ class MessageHistoryTableTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } diff --git a/PostboxTests/OrderStatisticTreeTests.swift b/PostboxTests/OrderStatisticTreeTests.swift index 10ce2c8eff..c4779434db 100644 --- a/PostboxTests/OrderStatisticTreeTests.swift +++ b/PostboxTests/OrderStatisticTreeTests.swift @@ -110,16 +110,16 @@ class OrderStatisticTreeTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil } func testOST() { - let tree = RBTree(rootData: 0) + /*let tree = RBTree(rootData: 0) for _ in 0 ..< 1000 { let key = Int(arc4random_uniform(UInt32(Int32.max - 1))) tree.insert(key) } - print("OST height: \(tree.depth())") + print("OST height: \(tree.depth())")*/ } } diff --git a/PostboxTests/RandomAccessResourceTests.swift b/PostboxTests/RandomAccessResourceTests.swift new file mode 100644 index 0000000000..b0f72a3065 --- /dev/null +++ b/PostboxTests/RandomAccessResourceTests.swift @@ -0,0 +1,413 @@ +import Foundation + +import UIKit +import XCTest + +import Postbox +@testable import Postbox + +import SwiftSignalKit + +func randomFilePath() -> String { + return NSTemporaryDirectory() + "\(arc4random())\(arc4random())" +} + +class RandomAccessResourceTests: XCTestCase { + override func setUp() { + super.setUp() + + self.continueAfterFailure = false + } + + override func tearDown() { + super.tearDown() + } + + func testCompleteAligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + //let size = 64 * 1024 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + let subdata = sampleData.subdata(with: NSRange(location: range.lowerBound, length: range.count)) + storeRange(RandomAccessResourceStoreRange(offset: range.lowerBound, data: subdata)) + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + while !blocks.isEmpty { + let arrayOffset = Int(arc4random_uniform(UInt32(blocks.count))) + var rangeEnd = blocks[arrayOffset] + var arrayOffsetEnd = arrayOffset + for i in arrayOffset + 1 ..< blocks.count { + if blocks[i] == rangeEnd + 1 { + rangeEnd = blocks[i] + arrayOffsetEnd = i + } else { + break + } + } + + let arrayOffsetCount = arrayOffsetEnd + 1 - arrayOffset + let selectedArrayOffsetCount = Int(arc4random_uniform(UInt32(arrayOffsetCount + 1))) + let selectedArrayOffsetEnd = arrayOffset + max(0, selectedArrayOffsetCount - 1) + + let range = (blocks[arrayOffset] * context.blockSize) ..< ((blocks[selectedArrayOffsetEnd] + 1) * context.blockSize) + blocks.removeSubrange(arrayOffset ..< (selectedArrayOffsetEnd + 1)) + + selectedRanges.append(Range(range)) + } + + selectedRanges.removeAll() + + selectedRanges.append(10092544..<10354688); selectedRanges.append(1310720..<7274496); selectedRanges.append(7340032..<7798784); selectedRanges.append(1179648..<1310720); selectedRanges.append(524288..<851968); selectedRanges.append(8060928..<9895936); selectedRanges.append(1114112..<1179648); selectedRanges.append(7929856..<8060928); selectedRanges.append(196608..<524288); selectedRanges.append(131072..<196608); selectedRanges.append(7798784..<7864320); selectedRanges.append(917504..<1114112); selectedRanges.append(65536..<131072); selectedRanges.append(7274496..<7340032); selectedRanges.append(0..<65536); selectedRanges.append(7864320..<7929856); selectedRanges.append(851968..<917504); selectedRanges.append(9961472..<10027008); selectedRanges.append(10354688..<10420224); selectedRanges.append(10027008..<10092544); selectedRanges.append(10420224..<10485760); selectedRanges.append(9895936..<9961472) + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var invocationCount = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + XCTAssert(subdata.count == range.count, "\(subdata.count) != \(range.count)") + invocationCount += 1 + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, range.count) + } + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(invocationCount == 1, "invocationCount != 1") + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testCompleteUnaligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + 123 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + let subdata = sampleData.subdata(with: NSRange(location: range.lowerBound, length: range.count)) + storeRange(RandomAccessResourceStoreRange(offset: range.lowerBound, data: subdata)) + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + var dataOffset = 0 + while dataOffset < size { + let partSize = min(size - dataOffset, Int(arc4random_uniform(1024 * 1024))) + selectedRanges.append(dataOffset ..< (dataOffset + partSize)) + dataOffset += partSize + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + if range.count != subdata.count { + print("\(subdata.count)") + } + XCTAssert(subdata.count == range.count) + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, subdata.count) + } + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreCompleteSubscriptionAligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + //let size = 64 * 1024 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: range.lowerBound, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + while !blocks.isEmpty { + let arrayOffset = Int(arc4random_uniform(UInt32(blocks.count))) + var rangeEnd = blocks[arrayOffset] + var arrayOffsetEnd = arrayOffset + for i in arrayOffset + 1 ..< blocks.count { + if blocks[i] == rangeEnd + 1 { + rangeEnd = blocks[i] + arrayOffsetEnd = i + } else { + break + } + } + + let arrayOffsetCount = arrayOffsetEnd + 1 - arrayOffset + let selectedArrayOffsetCount = Int(arc4random_uniform(UInt32(arrayOffsetCount + 1))) + let selectedArrayOffsetEnd = arrayOffset + max(0, selectedArrayOffsetCount - 1) + + let range = (blocks[arrayOffset] * context.blockSize) ..< ((blocks[selectedArrayOffsetEnd] + 1) * context.blockSize) + blocks.removeSubrange(arrayOffset ..< (selectedArrayOffsetEnd + 1)) + + selectedRanges.append(Range(range)) + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var invocations = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + XCTAssert(subdata.count == range.count) + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, range.count) + } + invocations += 1 + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(invocations == 1) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreCompleteSubscriptionUnaligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + 123 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: offset, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var selectedRanges: [Range] = [] + + var dataOffset = 0 + while dataOffset < size { + let partSize = min(size - dataOffset, Int(arc4random_uniform(1024 * 1024))) + selectedRanges.append(dataOffset ..< (dataOffset + partSize)) + dataOffset += partSize + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var invocations = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Complete, updated: { subdata in + XCTAssert(subdata.count == range.count) + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound), bytes, subdata.count) + } + invocations += 1 + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(invocations == 1) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreIncrementalSubscriptionAligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: offset, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var blocks: [Int] = [] + for i in 0 ..< context.blockCount { + blocks.append(i) + } + + var selectedRanges: [Range] = [] + + while !blocks.isEmpty { + let arrayOffset = Int(arc4random_uniform(UInt32(blocks.count))) + var rangeEnd = blocks[arrayOffset] + var arrayOffsetEnd = arrayOffset + for i in arrayOffset + 1 ..< blocks.count { + if blocks[i] == rangeEnd + 1 { + rangeEnd = blocks[i] + arrayOffsetEnd = i + } else { + break + } + } + + let arrayOffsetCount = arrayOffsetEnd + 1 - arrayOffset + let selectedArrayOffsetCount = Int(arc4random_uniform(UInt32(arrayOffsetCount + 1))) + let selectedArrayOffsetEnd = arrayOffset + max(0, selectedArrayOffsetCount - 1) + + let range = (blocks[arrayOffset] * context.blockSize) ..< ((blocks[selectedArrayOffsetEnd] + 1) * context.blockSize) + blocks.removeSubrange(arrayOffset ..< (selectedArrayOffsetEnd + 1)) + + selectedRanges.append(Range(range)) + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var offset = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Incremental, updated: { subdata in + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound + offset), bytes, subdata.count) + } + offset += subdata.count + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(offset == range.count) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } + + func testIncrementalStoreIncrementalSubscriptionUnaligned() { + let path = randomFilePath() + + let size = 10 * 1024 * 1024 + 123 + let sampleData = NSMutableData() + sampleData.length = size + arc4random_buf(sampleData.mutableBytes, size) + + var storeRange: (RandomAccessResourceStoreRange) -> Void = { _ in } + + let context = RandomAccessMediaResourceContext(path: path, size: size, fetchRange: { range in + var offset = range.lowerBound + while offset < range.upperBound { + let subdata = sampleData.subdata(with: NSRange(location: offset, length: min(range.upperBound - offset, 64 * 1024))) + storeRange(RandomAccessResourceStoreRange(offset: offset, data: subdata)) + + offset += 64 * 1024 + } + return EmptyDisposable + }) + + storeRange = { [weak context] range in + context?.storeRanges([range]) + } + + var selectedRanges: [Range] = [] + + selectedRanges = [0..<615697, 615697..<1040801] + + var dataOffset = 1040801 + while dataOffset < size { + let partSize = min(size - dataOffset, Int(arc4random_uniform(1024 * 1024))) + selectedRanges.append(dataOffset ..< (dataOffset + partSize)) + dataOffset += partSize + } + + print("\(selectedRanges)") + + let testData = NSMutableData() + testData.length = size + for range in selectedRanges { + var offset = 0 + let _ = context.addListenerForData(in: Range(range), mode: .Incremental, updated: { subdata in + subdata.withUnsafeBytes { (bytes: UnsafePointer) -> Void in + memcpy(testData.mutableBytes.advanced(by: range.lowerBound + offset), bytes, subdata.count) + } + offset += subdata.count + }) + let _ = context.addListenerForFetchedData(in: Range(range)) + XCTAssert(offset == range.count) + } + + XCTAssert(memcmp(testData.bytes, sampleData.bytes, size) == 0) + } +} diff --git a/PostboxTests/ReadStateTableTests.swift b/PostboxTests/ReadStateTableTests.swift index eea587f1aa..eb51113df4 100644 --- a/PostboxTests/ReadStateTableTests.swift +++ b/PostboxTests/ReadStateTableTests.swift @@ -110,7 +110,7 @@ class ReadStateTableTests: XCTestCase { self.historyMetadataTable = nil self.valueBox = nil - let _ = try? FileManager.default().removeItem(atPath: path!) + let _ = try? FileManager.default.removeItem(atPath: path!) self.path = nil }