ThePain/Swiftgram
1
0
Fork 0
mirror of https://github.com/Swiftgram/Telegram-iOS.git synced 2025-06-15 21:45:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Video Player V2

Browse Source
This commit is contained in:
Isaac 2024-12-27 20:07:37 +08:00
parent 03b4a57a59
commit b20cc376c4
25 changed files with 1724 additions and 712 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
submodules/FFMpegBinding/Public/FFMpegBinding/FFMpegAVCodec.h
View File

@ -4,7 +4,7 @@ NS_ASSUME_NONNULL_BEGIN
@interface FFMpegAVCodec : NSObject
+ (FFMpegAVCodec * _Nullable)findForId:(int)codecId;
+ (FFMpegAVCodec * _Nullable)findForId:(int)codecId preferHardwareAccelerationCapable:(bool)preferHardwareAccelerationCapable;
- (void *)impl;

1
submodules/FFMpegBinding/Public/FFMpegBinding/FFMpegAVCodecContext.h
View File

@ -25,6 +25,7 @@ typedef NS_ENUM(NSUInteger, FFMpegAVCodecContextReceiveResult)
- (bool)open;
- (bool)sendEnd;
- (void)setupHardwareAccelerationIfPossible;
- (FFMpegAVCodecContextReceiveResult)receiveIntoFrame:(FFMpegAVFrame *)frame;
- (void)flushBuffers;

6
submodules/FFMpegBinding/Public/FFMpegBinding/FFMpegAVFrame.h
View File

@ -12,6 +12,11 @@ typedef NS_ENUM(NSUInteger, FFMpegAVFramePixelFormat) {
FFMpegAVFramePixelFormatYUVA
};
typedef NS_ENUM(NSUInteger, FFMpegAVFrameNativePixelFormat) {
FFMpegAVFrameNativePixelFormatUnknown,
FFMpegAVFrameNativePixelFormatVideoToolbox
};
@interface FFMpegAVFrame : NSObject
@property (nonatomic, readonly) int32_t width;
@ -27,6 +32,7 @@ typedef NS_ENUM(NSUInteger, FFMpegAVFramePixelFormat) {
- (instancetype)initWithPixelFormat:(FFMpegAVFramePixelFormat)pixelFormat width:(int32_t)width height:(int32_t)height;
- (void *)impl;
- (FFMpegAVFrameNativePixelFormat)nativePixelFormat;
@end

21
submodules/FFMpegBinding/Sources/FFMpegAVCodec.m
View File

@ -18,7 +18,26 @@
return self;
}
+ (FFMpegAVCodec * _Nullable)findForId:(int)codecId {
+ (FFMpegAVCodec * _Nullable)findForId:(int)codecId preferHardwareAccelerationCapable:(_Bool)preferHardwareAccelerationCapable {
if (preferHardwareAccelerationCapable && codecId == AV_CODEC_ID_AV1) {
void *codecIterationState = nil;
while (true) {
AVCodec const *codec = av_codec_iterate(&codecIterationState);
if (!codec) {
break;
}
if (!av_codec_is_decoder(codec)) {
continue;
}
if (codec->id != codecId) {
continue;
}
if (strncmp(codec->name, "av1", 2) == 0) {
return [[FFMpegAVCodec alloc] initWithImpl:codec];
}
}
}
AVCodec const *codec = avcodec_find_decoder(codecId);
if (codec) {
return [[FFMpegAVCodec alloc] initWithImpl:codec];

9
submodules/FFMpegBinding/Sources/FFMpegAVCodecContext.m
View File

@ -6,6 +6,10 @@
#import "libavformat/avformat.h"
#import "libavcodec/avcodec.h"
static enum AVPixelFormat getPreferredPixelFormat(__unused AVCodecContext *ctx, __unused const enum AVPixelFormat *pix_fmts) {
return AV_PIX_FMT_VIDEOTOOLBOX;
}
@interface FFMpegAVCodecContext () {
FFMpegAVCodec *_codec;
AVCodecContext *_impl;
@ -61,6 +65,11 @@
return status == 0;
}
- (void)setupHardwareAccelerationIfPossible {
av_hwdevice_ctx_create(&_impl->hw_device_ctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nil, nil, 0);
_impl->get_format = getPreferredPixelFormat;
}
- (FFMpegAVCodecContextReceiveResult)receiveIntoFrame:(FFMpegAVFrame *)frame {
int status = avcodec_receive_frame(_impl, (AVFrame *)[frame impl]);
if (status == 0) {

11
submodules/FFMpegBinding/Sources/FFMpegAVFrame.m
View File

@ -64,6 +64,17 @@
return _impl->pts;
}
- (FFMpegAVFrameNativePixelFormat)nativePixelFormat {
switch (_impl->format) {
case AV_PIX_FMT_VIDEOTOOLBOX: {
return FFMpegAVFrameNativePixelFormatVideoToolbox;
}
default: {
return FFMpegAVFrameNativePixelFormatUnknown;
}
}
}
- (int64_t)duration {
#if LIBAVFORMAT_VERSION_MAJOR >= 59
return _impl->duration;

2
submodules/GalleryUI/Sources/GalleryController.swift
View File

@ -259,7 +259,7 @@ public func galleryItemForEntry(
}
if isHLS {
content = HLSVideoContent(id: .message(message.stableId, file.fileId), userLocation: .peer(message.id.peerId), fileReference: .message(message: MessageReference(message), media: file), streamVideo: streamVideos, loopVideo: loopVideos, codecConfiguration: HLSCodecConfiguration(context: context))
content = HLSVideoContent(id: .message(message.stableId, file.fileId), userLocation: .peer(message.id.peerId), fileReference: .message(message: MessageReference(message), media: file), streamVideo: streamVideos, loopVideo: loopVideos, autoFetchFullSizeThumbnail: true, codecConfiguration: HLSCodecConfiguration(context: context))
} else {
content = NativeVideoContent(id: .message(message.stableId, file.fileId), userLocation: .peer(message.id.peerId), fileReference: .message(message: MessageReference(message), media: file), imageReference: mediaImage.flatMap({ ImageMediaReference.message(message: MessageReference(message), media: $0) }), streamVideo: .conservative, loopVideo: loopVideos, tempFilePath: tempFilePath, captureProtected: captureProtected, storeAfterDownload: generateStoreAfterDownload?(message, file))
}

2
submodules/Media/ConvertOpusToAAC/Sources/ConvertOpusToAAC.swift
View File

@ -10,7 +10,7 @@ public func convertOpusToAAC(sourcePath: String, allocateTempFile: @escaping ()
queue.async {
do {
let audioSource = SoftwareAudioSource(path: sourcePath, focusedPart: nil)
let audioSource = SoftwareAudioSource(path: sourcePath)
let outputPath = allocateTempFile()

1
submodules/MediaPlayer/BUILD
View File

@ -21,6 +21,7 @@ swift_library(
"//submodules/YuvConversion:YuvConversion",
"//submodules/Utils/RangeSet:RangeSet",
"//submodules/TextFormat:TextFormat",
"//submodules/ManagedFile",
],
visibility = [
"//visibility:public",

80
submodules/MediaPlayer/Sources/ChunkMediaPlayer.swift
View File

@ -9,7 +9,7 @@ import TelegramAudio
public final class ChunkMediaPlayerPart {
public enum Id: Hashable {
case tempFile(path: String)
case directFile(path: String, audio: DirectStream?, video: DirectStream?)
case directStream
}
public struct DirectStream: Hashable {
@ -26,51 +26,29 @@ public final class ChunkMediaPlayerPart {
}
}
public enum Content {
public final class TempFile {
public let file: TempBoxFile
public init(file: TempBoxFile) {
self.file = file
}
deinit {
TempBox.shared.dispose(self.file)
}
public final class TempFile {
public let file: TempBoxFile
public init(file: TempBoxFile) {
self.file = file
}
public final class FFMpegDirectFile {
public let path: String
public let audio: DirectStream?
public let video: DirectStream?
public init(path: String, audio: DirectStream?, video: DirectStream?) {
self.path = path
self.audio = audio
self.video = video
}
deinit {
TempBox.shared.dispose(self.file)
}
case tempFile(TempFile)
case directFile(FFMpegDirectFile)
}
public let startTime: Double
public let endTime: Double
public let content: Content
public let content: TempFile
public let clippedStartTime: Double?
public let codecName: String?
public var id: Id {
switch self.content {
case let .tempFile(tempFile):
return .tempFile(path: tempFile.file.path)
case let .directFile(directFile):
return .directFile(path: directFile.path, audio: directFile.audio, video: directFile.video)
}
return .tempFile(path: self.content.file.path)
}
public init(startTime: Double, clippedStartTime: Double? = nil, endTime: Double, content: Content, codecName: String?) {
public init(startTime: Double, clippedStartTime: Double? = nil, endTime: Double, content: TempFile, codecName: String?) {
self.startTime = startTime
self.clippedStartTime = clippedStartTime
self.endTime = endTime
@ -81,23 +59,47 @@ public final class ChunkMediaPlayerPart {
public final class ChunkMediaPlayerPartsState {
public final class DirectReader {
public final class Impl {
public let video: MediaDataReader?
public let audio: MediaDataReader?
public struct Stream {
public let mediaBox: MediaBox
public let resource: MediaResource
public let size: Int64
public let index: Int
public let seek: (streamIndex: Int, pts: Int64)
public let maxReadablePts: (streamIndex: Int, pts: Int64, isEnded: Bool)?
public let codecName: String?
public init(video: MediaDataReader?, audio: MediaDataReader?) {
public init(mediaBox: MediaBox, resource: MediaResource, size: Int64, index: Int, seek: (streamIndex: Int, pts: Int64), maxReadablePts: (streamIndex: Int, pts: Int64, isEnded: Bool)?, codecName: String?) {
self.mediaBox = mediaBox
self.resource = resource
self.size = size
self.index = index
self.seek = seek
self.maxReadablePts = maxReadablePts
self.codecName = codecName
}
}
public final class Impl {
public let video: Stream?
public let audio: Stream?
public init(video: Stream?, audio: Stream?) {
self.video = video
self.audio = audio
}
}
public let id: Int
public let seekPosition: Double
public let availableUntilPosition: Double
public let impl: QueueLocalObject<Impl>
public let bufferedUntilEnd: Bool
public let impl: Impl?
public init(seekPosition: Double, availableUntilPosition: Double, impl: QueueLocalObject<Impl>) {
public init(id: Int, seekPosition: Double, availableUntilPosition: Double, bufferedUntilEnd: Bool, impl: Impl?) {
self.id = id
self.seekPosition = seekPosition
self.availableUntilPosition = availableUntilPosition
self.bufferedUntilEnd = bufferedUntilEnd
self.impl = impl
}
}

900
submodules/MediaPlayer/Sources/ChunkMediaPlayerDirectFetchSourceImpl.swift
View File

@ -6,394 +6,457 @@ import TelegramCore
import FFMpegBinding
import RangeSet
private final class FFMpegMediaFrameExtractContext {
let fd: Int32
var readPosition: Int = 0
let size: Int
private func FFMpegLookaheadReader_readPacketCallback(userData: UnsafeMutableRawPointer?, buffer: UnsafeMutablePointer<UInt8>?, bufferSize: Int32) -> Int32 {
let context = Unmanaged<FFMpegLookaheadReader>.fromOpaque(userData!).takeUnretainedValue()
var accessedRanges = RangeSet<Int>()
var maskRanges: RangeSet<Int>?
var recordAccessedRanges = false
let readCount = min(256 * 1024, Int64(bufferSize))
let requestRange: Range<Int64> = context.readingOffset ..< (context.readingOffset + readCount)
init(fd: Int32, size: Int) {
self.fd = fd
self.size = size
}
}
private func FFMpegMediaFrameExtractContextReadPacketCallback(userData: UnsafeMutableRawPointer?, buffer: UnsafeMutablePointer<UInt8>?, bufferSize: Int32) -> Int32 {
let context = Unmanaged<FFMpegMediaFrameExtractContext>.fromOpaque(userData!).takeUnretainedValue()
if context.recordAccessedRanges {
context.accessedRanges.insert(contentsOf: context.readPosition ..< (context.readPosition + Int(bufferSize)))
var fetchedData: Data?
let fetchDisposable = MetaDisposable()
let semaphore = DispatchSemaphore(value: 0)
let disposable = context.params.getDataInRange(requestRange, { data in
if let data {
fetchedData = data
semaphore.signal()
}
})
var isCancelled = false
let cancelDisposable = context.params.cancel.start(next: { _ in
isCancelled = true
semaphore.signal()
})
semaphore.wait()
if isCancelled {
context.isCancelled = true
}
let result: Int
if let maskRanges = context.maskRanges {
let readRange = context.readPosition ..< (context.readPosition + Int(bufferSize))
let _ = maskRanges
let _ = readRange
result = read(context.fd, buffer, Int(bufferSize))
disposable.dispose()
cancelDisposable.dispose()
fetchDisposable.dispose()
if let fetchedData = fetchedData {
fetchedData.withUnsafeBytes { byteBuffer -> Void in
guard let bytes = byteBuffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else {
return
}
memcpy(buffer, bytes, fetchedData.count)
}
let fetchedCount = Int32(fetchedData.count)
context.setReadingOffset(offset: context.readingOffset + Int64(fetchedCount))
if fetchedCount == 0 {
return FFMPEG_CONSTANT_AVERROR_EOF
}
return fetchedCount
} else {
result = read(context.fd, buffer, Int(bufferSize))
}
context.readPosition += Int(bufferSize)
if result == 0 {
return FFMPEG_CONSTANT_AVERROR_EOF
}
return Int32(result)
}
private func FFMpegMediaFrameExtractContextSeekCallback(userData: UnsafeMutableRawPointer?, offset: Int64, whence: Int32) -> Int64 {
let context = Unmanaged<FFMpegMediaFrameExtractContext>.fromOpaque(userData!).takeUnretainedValue()
private func FFMpegLookaheadReader_seekCallback(userData: UnsafeMutableRawPointer?, offset: Int64, whence: Int32) -> Int64 {
let context = Unmanaged<FFMpegLookaheadReader>.fromOpaque(userData!).takeUnretainedValue()
if (whence & FFMPEG_AVSEEK_SIZE) != 0 {
return Int64(context.size)
return context.params.size
} else {
context.readPosition = Int(offset)
lseek(context.fd, off_t(offset), SEEK_SET)
context.setReadingOffset(offset: offset)
return offset
}
}
private struct FFMpegFrameSegment {
struct Stream {
let index: Int
let startPts: CMTime
let startPosition: Int64
var endPts: CMTime
var endPosition: Int64
var duration: Double
}
var audio: Stream?
var video: Stream?
init() {
}
mutating func addFrame(isVideo: Bool, index: Int, pts: CMTime, duration: Double, position: Int64, size: Int64) {
if var stream = isVideo ? self.video : self.audio {
stream.endPts = pts
stream.duration += duration
stream.endPosition = max(stream.endPosition, position + size)
if isVideo {
self.video = stream
} else {
self.audio = stream
}
} else {
let stream = Stream(index: index, startPts: pts, startPosition: position, endPts: pts, endPosition: position + size, duration: duration)
if isVideo {
self.video = stream
} else {
self.audio = stream
}
}
}
private func range(_ outer: Range<Int64>, fullyContains inner: Range<Int64>) -> Bool {
return inner.lowerBound >= outer.lowerBound && inner.upperBound <= outer.upperBound
}
private final class FFMpegFrameSegmentInfo {
let headerAccessRanges: RangeSet<Int>
let segments: [FFMpegFrameSegment]
private final class FFMpegLookaheadReader {
let params: FFMpegLookaheadThread.Params
init(headerAccessRanges: RangeSet<Int>, segments: [FFMpegFrameSegment]) {
self.headerAccessRanges = headerAccessRanges
self.segments = segments
}
}
private func extractFFMpegFrameSegmentInfo(path: String) -> FFMpegFrameSegmentInfo? {
let _ = FFMpegMediaFrameSourceContextHelpers.registerFFMpegGlobals
var avIoContext: FFMpegAVIOContext?
var avFormatContext: FFMpegAVFormatContext?
var s = stat()
stat(path, &s)
let size = Int32(s.st_size)
var audioStream: FFMpegFileReader.StreamInfo?
var videoStream: FFMpegFileReader.StreamInfo?
let fd = open(path, O_RDONLY, S_IRUSR)
if fd < 0 {
return nil
}
defer {
close(fd)
}
var seekInfo: FFMpegLookaheadThread.State.Seek?
var maxReadPts: FFMpegLookaheadThread.State.Seek?
var audioStreamState: FFMpegLookaheadThread.StreamState?
var videoStreamState: FFMpegLookaheadThread.StreamState?
let avFormatContext = FFMpegAVFormatContext()
let ioBufferSize = 32 * 1024
var reportedState: FFMpegLookaheadThread.State?
let context = FFMpegMediaFrameExtractContext(fd: fd, size: Int(size))
context.recordAccessedRanges = true
var readingOffset: Int64 = 0
var isCancelled: Bool = false
var isEnded: Bool = false
guard let avIoContext = FFMpegAVIOContext(bufferSize: Int32(ioBufferSize), opaqueContext: Unmanaged.passUnretained(context).toOpaque(), readPacket: FFMpegMediaFrameExtractContextReadPacketCallback, writePacket: nil, seek: FFMpegMediaFrameExtractContextSeekCallback, isSeekable: true) else {
return nil
}
private var currentFetchRange: Range<Int64>?
private var currentFetchDisposable: Disposable?
avFormatContext.setIO(avIoContext)
var currentTimestamp: Double?
if !avFormatContext.openInput(withDirectFilePath: nil) {
return nil
}
if !avFormatContext.findStreamInfo() {
return nil
}
var audioStream: FFMpegMediaInfo.Info?
var videoStream: FFMpegMediaInfo.Info?
for typeIndex in 0 ..< 2 {
let isVideo = typeIndex == 0
init?(params: FFMpegLookaheadThread.Params) {
self.params = params
for streamIndexNumber in avFormatContext.streamIndices(for: isVideo ? FFMpegAVFormatStreamTypeVideo : FFMpegAVFormatStreamTypeAudio) {
let streamIndex = streamIndexNumber.int32Value
if avFormatContext.isAttachedPic(atStreamIndex: streamIndex) {
continue
}
let fpsAndTimebase = avFormatContext.fpsAndTimebase(forStreamIndex: streamIndex, defaultTimeBase: CMTimeMake(value: 1, timescale: 40000))
let (fps, timebase) = (fpsAndTimebase.fps, fpsAndTimebase.timebase)
let startTime: CMTime
let rawStartTime = avFormatContext.startTime(atStreamIndex: streamIndex)
if rawStartTime == Int64(bitPattern: 0x8000000000000000 as UInt64) {
startTime = CMTime(value: 0, timescale: timebase.timescale)
} else {
startTime = CMTimeMake(value: rawStartTime, timescale: timebase.timescale)
}
var duration = CMTimeMake(value: avFormatContext.duration(atStreamIndex: streamIndex), timescale: timebase.timescale)
duration = CMTimeMaximum(CMTime(value: 0, timescale: duration.timescale), CMTimeSubtract(duration, startTime))
var codecName: String?
let codecId = avFormatContext.codecId(atStreamIndex: streamIndex)
if codecId == FFMpegCodecIdMPEG4 {
codecName = "mpeg4"
} else if codecId == FFMpegCodecIdH264 {
codecName = "h264"
} else if codecId == FFMpegCodecIdHEVC {
codecName = "hevc"
} else if codecId == FFMpegCodecIdAV1 {
codecName = "av1"
} else if codecId == FFMpegCodecIdVP9 {
codecName = "vp9"
} else if codecId == FFMpegCodecIdVP8 {
codecName = "vp8"
}
let info = FFMpegMediaInfo.Info(
index: Int(streamIndex),
timescale: timebase.timescale,
startTime: startTime,
duration: duration,
fps: fps,
codecName: codecName
)
if isVideo {
videoStream = info
} else {
audioStream = info
}
let ioBufferSize = 64 * 1024
guard let avIoContext = FFMpegAVIOContext(bufferSize: Int32(ioBufferSize), opaqueContext: Unmanaged.passUnretained(self).toOpaque(), readPacket: FFMpegLookaheadReader_readPacketCallback, writePacket: nil, seek: FFMpegLookaheadReader_seekCallback, isSeekable: true) else {
return nil
}
}
var segments: [FFMpegFrameSegment] = []
let maxSegmentDuration: Double = 5.0
if let videoStream {
let indexEntryCount = avFormatContext.numberOfIndexEntries(atStreamIndex: Int32(videoStream.index))
self.avIoContext = avIoContext
if indexEntryCount > 0 {
let frameDuration = 1.0 / videoStream.fps.seconds
var indexEntry = FFMpegAVIndexEntry()
for i in 0 ..< indexEntryCount {
if !avFormatContext.fillIndexEntry(atStreamIndex: Int32(videoStream.index), entryIndex: Int32(i), outEntry: &indexEntry) {
let avFormatContext = FFMpegAVFormatContext()
avFormatContext.setIO(avIoContext)
if !avFormatContext.openInput(withDirectFilePath: nil) {
return nil
}
if !avFormatContext.findStreamInfo() {
return nil
}
self.avFormatContext = avFormatContext
var audioStream: FFMpegFileReader.StreamInfo?
var videoStream: FFMpegFileReader.StreamInfo?
for streamType in 0 ..< 2 {
let isVideo = streamType == 0
for streamIndexNumber in avFormatContext.streamIndices(for: isVideo ? FFMpegAVFormatStreamTypeVideo : FFMpegAVFormatStreamTypeAudio) {
let streamIndex = streamIndexNumber.int32Value
if avFormatContext.isAttachedPic(atStreamIndex: streamIndex) {
continue
}
let packetPts = CMTime(value: indexEntry.timestamp, timescale: videoStream.timescale)
//print("index: \(packetPts.seconds), isKeyframe: \(indexEntry.isKeyframe), position: \(indexEntry.pos), size: \(indexEntry.size)")
let codecId = avFormatContext.codecId(atStreamIndex: streamIndex)
var startNewSegment = segments.isEmpty
if indexEntry.isKeyframe {
if segments.isEmpty {
startNewSegment = true
} else if let video = segments[segments.count - 1].video {
if packetPts.seconds - video.startPts.seconds > maxSegmentDuration {
startNewSegment = true
}
}
}
let fpsAndTimebase = avFormatContext.fpsAndTimebase(forStreamIndex: streamIndex, defaultTimeBase: CMTimeMake(value: 1, timescale: 40000))
let (fps, timebase) = (fpsAndTimebase.fps, fpsAndTimebase.timebase)
if startNewSegment {
segments.append(FFMpegFrameSegment())
let startTime: CMTime
let rawStartTime = avFormatContext.startTime(atStreamIndex: streamIndex)
if rawStartTime == Int64(bitPattern: 0x8000000000000000 as UInt64) {
startTime = CMTime(value: 0, timescale: timebase.timescale)
} else {
startTime = CMTimeMake(value: rawStartTime, timescale: timebase.timescale)
}
segments[segments.count - 1].addFrame(isVideo: true, index: videoStream.index, pts: packetPts, duration: frameDuration, position: indexEntry.pos, size: Int64(indexEntry.size))
}
if !segments.isEmpty, let video = segments[segments.count - 1].video {
if video.endPts.seconds + 1.0 / videoStream.fps.seconds + 0.001 < videoStream.duration.seconds {
segments[segments.count - 1].video?.duration = videoStream.duration.seconds - video.startPts.seconds
segments[segments.count - 1].video?.endPts = videoStream.duration
var duration = CMTimeMake(value: avFormatContext.duration(atStreamIndex: streamIndex), timescale: timebase.timescale)
duration = CMTimeMaximum(CMTime(value: 0, timescale: duration.timescale), CMTimeSubtract(duration, startTime))
//let metrics = avFormatContext.metricsForStream(at: streamIndex)
//let rotationAngle: Double = metrics.rotationAngle
//let aspect = Double(metrics.width) / Double(metrics.height)
let stream = FFMpegFileReader.StreamInfo(
index: streamIndexNumber.intValue,
codecId: codecId,
startTime: startTime,
duration: duration,
timeBase: timebase.value,
timeScale: timebase.timescale,
fps: fps
)
if isVideo {
videoStream = stream
} else {
audioStream = stream
}
}
}
self.audioStream = audioStream
self.videoStream = videoStream
if let preferredStream = self.videoStream ?? self.audioStream {
let pts = CMTimeMakeWithSeconds(params.seekToTimestamp, preferredTimescale: preferredStream.timeScale)
self.seekInfo = FFMpegLookaheadThread.State.Seek(streamIndex: preferredStream.index, pts: pts.value)
avFormatContext.seekFrame(forStreamIndex: Int32(preferredStream.index), pts: pts.value, positionOnKeyframe: true)
}
self.updateCurrentTimestamp()
}
if let audioStream {
let indexEntryCount = avFormatContext.numberOfIndexEntries(atStreamIndex: Int32(audioStream.index))
if indexEntryCount > 0 {
var minSegmentIndex = 0
var minSegmentStartTime: Double = -100000.0
let frameDuration = 1.0 / audioStream.fps.seconds
var indexEntry = FFMpegAVIndexEntry()
for i in 0 ..< indexEntryCount {
if !avFormatContext.fillIndexEntry(atStreamIndex: Int32(audioStream.index), entryIndex: Int32(i), outEntry: &indexEntry) {
continue
}
let packetPts = CMTime(value: indexEntry.timestamp, timescale: audioStream.timescale)
//print("index: \(packetPts.value), timestamp: \(packetPts.seconds), isKeyframe: \(indexEntry.isKeyframe), position: \(indexEntry.pos), size: \(indexEntry.size)")
if videoStream != nil {
for i in minSegmentIndex ..< segments.count {
if let video = segments[i].video {
if minSegmentStartTime <= packetPts.seconds && video.endPts.seconds >= packetPts.seconds {
segments[i].addFrame(isVideo: false, index: audioStream.index, pts: packetPts, duration: frameDuration, position: indexEntry.pos, size: Int64(indexEntry.size))
if minSegmentIndex != i {
minSegmentIndex = i
minSegmentStartTime = video.startPts.seconds
}
break
}
}
deinit {
self.currentFetchDisposable?.dispose()
}
func setReadingOffset(offset: Int64) {
self.readingOffset = offset
let readRange: Range<Int64> = offset ..< (offset + 512 * 1024)
if !self.params.isDataCachedInRange(readRange) {
if let currentFetchRange = self.currentFetchRange {
if currentFetchRange.overlaps(readRange) {
if !range(currentFetchRange, fullyContains: readRange) {
self.setFetchRange(range: currentFetchRange.lowerBound ..< max(currentFetchRange.upperBound, readRange.upperBound + 2 * 1024 * 1024))
}
} else {
if segments.isEmpty {
segments.append(FFMpegFrameSegment())
}
segments[segments.count - 1].addFrame(isVideo: false, index: audioStream.index, pts: packetPts, duration: frameDuration, position: indexEntry.pos, size: Int64(indexEntry.size))
self.setFetchRange(range: offset ..< (offset + 2 * 1024 * 1024))
}
}
}
if !segments.isEmpty, let audio = segments[segments.count - 1].audio {
if audio.endPts.seconds + 0.001 < audioStream.duration.seconds {
segments[segments.count - 1].audio?.duration = audioStream.duration.seconds - audio.startPts.seconds
segments[segments.count - 1].audio?.endPts = audioStream.duration
} else {
self.setFetchRange(range: offset ..< (offset + 2 * 1024 * 1024))
}
}
}
let headerAccessRanges = context.accessedRanges
private func setFetchRange(range: Range<Int64>) {
if self.currentFetchRange != range {
self.currentFetchRange = range
self.currentFetchDisposable?.dispose()
self.currentFetchDisposable = self.params.fetchInRange(range)
}
}
for i in 1 ..< segments.count {
let segment = segments[i]
func updateCurrentTimestamp() {
self.currentTimestamp = self.params.currentTimestamp.with({ $0 })
if let video = segment.video {
context.maskRanges = headerAccessRanges
context.maskRanges?.insert(contentsOf: Int(video.startPosition) ..< Int(video.endPosition))
self.updateReadIfNeeded()
}
private func updateReadIfNeeded() {
guard let avFormatContext = self.avFormatContext else {
return
}
guard let currentTimestamp = self.currentTimestamp else {
return
}
let maxPtsSeconds = max(self.params.seekToTimestamp, currentTimestamp) + 10.0
var currentAudioPtsSecondsAdvanced: Double = 0.0
var currentVideoPtsSecondsAdvanced: Double = 0.0
let packet = FFMpegPacket()
while !self.isCancelled && !self.isEnded {
var audioAlreadyRead: Bool = false
var videoAlreadyRead: Bool = false
context.accessedRanges = RangeSet()
context.recordAccessedRanges = true
avFormatContext.seekFrame(forStreamIndex: Int32(video.index), byteOffset: video.startPosition)
let packet = FFMpegPacket()
while true {
if !avFormatContext.readFrame(into: packet) {
break
}
if Int(packet.streamIndex) == video.index {
let packetPts = CMTime(value: packet.pts, timescale: video.startPts.timescale)
if packetPts.value >= video.endPts.value {
break
}
if let audioStreamState = self.audioStreamState {
if audioStreamState.readableToTime.seconds >= maxPtsSeconds {
audioAlreadyRead = true
}
} else if self.audioStream == nil {
audioAlreadyRead = true
}
print("Segment \(i): \(video.startPosition) ..< \(video.endPosition) accessed \(context.accessedRanges.ranges)")
if let videoStreamState = self.videoStreamState {
if videoStreamState.readableToTime.seconds >= maxPtsSeconds {
videoAlreadyRead = true
}
} else if self.videoStream == nil {
videoAlreadyRead = true
}
if audioAlreadyRead && videoAlreadyRead {
break
}
if !avFormatContext.readFrame(into: packet) {
self.isEnded = true
break
}
self.maxReadPts = FFMpegLookaheadThread.State.Seek(streamIndex: Int(packet.streamIndex), pts: packet.pts)
if let audioStream = self.audioStream, Int(packet.streamIndex) == audioStream.index {
let pts = CMTimeMake(value: packet.pts, timescale: audioStream.timeScale)
if let audioStreamState = self.audioStreamState {
currentAudioPtsSecondsAdvanced += pts.seconds - audioStreamState.readableToTime.seconds
}
self.audioStreamState = FFMpegLookaheadThread.StreamState(
info: audioStream,
readableToTime: pts
)
} else if let videoStream = self.videoStream, Int(packet.streamIndex) == videoStream.index {
let pts = CMTimeMake(value: packet.pts, timescale: videoStream.timeScale)
if let videoStreamState = self.videoStreamState {
currentVideoPtsSecondsAdvanced += pts.seconds - videoStreamState.readableToTime.seconds
}
self.videoStreamState = FFMpegLookaheadThread.StreamState(
info: videoStream,
readableToTime: pts
)
}
if min(currentAudioPtsSecondsAdvanced, currentVideoPtsSecondsAdvanced) >= 0.1 {
self.reportStateIfNeeded()
}
}
self.reportStateIfNeeded()
}
private func reportStateIfNeeded() {
guard let seekInfo = self.seekInfo else {
return
}
var stateIsFullyInitialised = true
if self.audioStream != nil && self.audioStreamState == nil {
stateIsFullyInitialised = false
}
if self.videoStream != nil && self.videoStreamState == nil {
stateIsFullyInitialised = false
}
let state = FFMpegLookaheadThread.State(
seek: seekInfo,
maxReadablePts: self.maxReadPts,
audio: (stateIsFullyInitialised && self.maxReadPts != nil) ? self.audioStreamState : nil,
video: (stateIsFullyInitialised && self.maxReadPts != nil) ? self.videoStreamState : nil,
isEnded: self.isEnded
)
if self.reportedState != state {
self.reportedState = state
self.params.updateState(state)
}
}
}
private final class FFMpegLookaheadThread: NSObject {
struct StreamState: Equatable {
let info: FFMpegFileReader.StreamInfo
let readableToTime: CMTime
init(info: FFMpegFileReader.StreamInfo, readableToTime: CMTime) {
self.info = info
self.readableToTime = readableToTime
}
}
/*{
if let videoStream {
avFormatContext.seekFrame(forStreamIndex: Int32(videoStream.index), pts: 0, positionOnKeyframe: true)
struct State: Equatable {
struct Seek: Equatable {
var streamIndex: Int
var pts: Int64
let packet = FFMpegPacket()
while true {
if !avFormatContext.readFrame(into: packet) {
break
}
if Int(packet.streamIndex) == videoStream.index {
let packetPts = CMTime(value: packet.pts, timescale: videoStream.timescale)
let packetDuration = CMTime(value: packet.duration, timescale: videoStream.timescale)
var startNewSegment = segments.isEmpty
if packet.isKeyframe {
if segments.isEmpty {
startNewSegment = true
} else if let video = segments[segments.count - 1].video {
if packetPts.seconds - video.startPts.seconds > maxSegmentDuration {
startNewSegment = true
}
}
}
if startNewSegment {
segments.append(FFMpegFrameSegment())
}
segments[segments.count - 1].addFrame(isVideo: true, index: Int(packet.streamIndex), pts: packetPts, duration: packetDuration.seconds)
}
init(streamIndex: Int, pts: Int64) {
self.streamIndex = streamIndex
self.pts = pts
}
}
if let audioStream {
avFormatContext.seekFrame(forStreamIndex: Int32(audioStream.index), pts: 0, positionOnKeyframe: true)
var minSegmentIndex = 0
let packet = FFMpegPacket()
while true {
if !avFormatContext.readFrame(into: packet) {
break
}
if Int(packet.streamIndex) == audioStream.index {
let packetPts = CMTime(value: packet.pts, timescale: audioStream.timescale)
let packetDuration = CMTime(value: packet.duration, timescale: audioStream.timescale)
if videoStream != nil {
for i in minSegmentIndex ..< segments.count {
if let video = segments[i].video {
if video.startPts.seconds <= packetPts.seconds && video.endPts.seconds >= packetPts.seconds {
segments[i].addFrame(isVideo: false, index: Int(audioStream.index), pts: packetPts, duration: packetDuration.seconds)
minSegmentIndex = i
break
}
}
}
} else {
if segments.isEmpty {
segments.append(FFMpegFrameSegment())
}
segments[segments.count - 1].addFrame(isVideo: false, index: Int(packet.streamIndex), pts: packetPts, duration: packetDuration.seconds)
}
}
let seek: Seek
let maxReadablePts: Seek?
let audio: StreamState?
let video: StreamState?
let isEnded: Bool
init(seek: Seek, maxReadablePts: Seek?, audio: StreamState?, video: StreamState?, isEnded: Bool) {
self.seek = seek
self.maxReadablePts = maxReadablePts
self.audio = audio
self.video = video
self.isEnded = isEnded
}
}
final class Params: NSObject {
let seekToTimestamp: Double
let updateState: (State) -> Void
let fetchInRange: (Range<Int64>) -> Disposable
let getDataInRange: (Range<Int64>, @escaping (Data?) -> Void) -> Disposable
let isDataCachedInRange: (Range<Int64>) -> Bool
let size: Int64
let cancel: Signal<Void, NoError>
let currentTimestamp: Atomic<Double?>
init(
seekToTimestamp: Double,
updateState: @escaping (State) -> Void,
fetchInRange: @escaping (Range<Int64>) -> Disposable,
getDataInRange: @escaping (Range<Int64>, @escaping (Data?) -> Void) -> Disposable,
isDataCachedInRange: @escaping (Range<Int64>) -> Bool,
size: Int64,
cancel: Signal<Void, NoError>,
currentTimestamp: Atomic<Double?>
) {
self.seekToTimestamp = seekToTimestamp
self.updateState = updateState
self.fetchInRange = fetchInRange
self.getDataInRange = getDataInRange
self.isDataCachedInRange = isDataCachedInRange
self.size = size
self.cancel = cancel
self.currentTimestamp = currentTimestamp
}
}
@objc static func entryPoint(_ params: Params) {
let runLoop = RunLoop.current
let timer = Timer(fireAt: .distantFuture, interval: 0.0, target: FFMpegLookaheadThread.self, selector: #selector(FFMpegLookaheadThread.none), userInfo: nil, repeats: false)
runLoop.add(timer, forMode: .common)
Thread.current.threadDictionary["FFMpegLookaheadThread_reader"] = FFMpegLookaheadReader(params: params)
while true {
runLoop.run(mode: .default, before: .distantFuture)
if Thread.current.threadDictionary["FFMpegLookaheadThread_stop"] != nil {
break
}
}
}*/
Thread.current.threadDictionary.removeObject(forKey: "FFMpegLookaheadThread_params")
}
/*for i in 0 ..< segments.count {
print("Segment \(i):\n video \(segments[i].video?.startPts.seconds ?? -1.0) ... \(segments[i].video?.endPts.seconds ?? -1.0)\n audio \(segments[i].audio?.startPts.seconds ?? -1.0) ... \(segments[i].audio?.endPts.seconds ?? -1.0)")
}*/
@objc static func none() {
}
return FFMpegFrameSegmentInfo(
headerAccessRanges: context.accessedRanges,
segments: segments
)
@objc static func stop() {
Thread.current.threadDictionary["FFMpegLookaheadThread_stop"] = "true"
}
@objc static func updateCurrentTimestamp() {
if let reader = Thread.current.threadDictionary["FFMpegLookaheadThread_reader"] as? FFMpegLookaheadReader {
reader.updateCurrentTimestamp()
}
}
}
private final class FFMpegLookahead {
private let cancel = Promise<Void>()
private let currentTimestamp = Atomic<Double?>(value: nil)
private let thread: Thread
init(
seekToTimestamp: Double,
updateState: @escaping (FFMpegLookaheadThread.State) -> Void,
fetchInRange: @escaping (Range<Int64>) -> Disposable,
getDataInRange: @escaping (Range<Int64>, @escaping (Data?) -> Void) -> Disposable,
isDataCachedInRange: @escaping (Range<Int64>) -> Bool,
size: Int64
) {
self.thread = Thread(
target: FFMpegLookaheadThread.self,
selector: #selector(FFMpegLookaheadThread.entryPoint(_:)),
object: FFMpegLookaheadThread.Params(
seekToTimestamp: seekToTimestamp,
updateState: updateState,
fetchInRange: fetchInRange,
getDataInRange: getDataInRange,
isDataCachedInRange: isDataCachedInRange,
size: size,
cancel: self.cancel.get(),
currentTimestamp: self.currentTimestamp
)
)
self.thread.name = "FFMpegLookahead"
self.thread.start()
}
deinit {
self.cancel.set(.single(Void()))
FFMpegLookaheadThread.self.perform(#selector(FFMpegLookaheadThread.stop), on: self.thread, with: nil, waitUntilDone: false)
}
func updateCurrentTimestamp(timestamp: Double) {
let _ = self.currentTimestamp.swap(timestamp)
FFMpegLookaheadThread.self.perform(#selector(FFMpegLookaheadThread.updateCurrentTimestamp), on: self.thread, with: timestamp as NSNumber, waitUntilDone: false)
}
}
final class ChunkMediaPlayerDirectFetchSourceImpl: ChunkMediaPlayerSourceImpl {
private let dataQueue: Queue
private let resource: ChunkMediaPlayerV2.SourceDescription.ResourceDescription
private let partsStateValue = Promise<ChunkMediaPlayerPartsState>()
@ -402,10 +465,12 @@ final class ChunkMediaPlayerDirectFetchSourceImpl: ChunkMediaPlayerSourceImpl {
}
private var completeFetchDisposable: Disposable?
private var dataDisposable: Disposable?
init(dataQueue: Queue, resource: ChunkMediaPlayerV2.SourceDescription.ResourceDescription) {
self.dataQueue = dataQueue
private var seekTimestamp: Double?
private var currentLookaheadId: Int = 0
private var lookahead: FFMpegLookahead?
init(resource: ChunkMediaPlayerV2.SourceDescription.ResourceDescription) {
self.resource = resource
if resource.fetchAutomatically {
@ -418,71 +483,134 @@ final class ChunkMediaPlayerDirectFetchSourceImpl: ChunkMediaPlayerSourceImpl {
preferBackgroundReferenceRevalidation: true
).startStrict()
}
self.dataDisposable = (resource.postbox.mediaBox.resourceData(resource.reference.resource)
|> deliverOnMainQueue).startStrict(next: { [weak self] data in
guard let self else {
return
}
if data.complete {
if let mediaInfo = extractFFMpegMediaInfo(path: data.path), let mainTrack = mediaInfo.audio ?? mediaInfo.video, let segmentInfo = extractFFMpegFrameSegmentInfo(path: data.path) {
var parts: [ChunkMediaPlayerPart] = []
for segment in segmentInfo.segments {
guard let mainStream = segment.video ?? segment.audio else {
assertionFailure()
continue
}
parts.append(ChunkMediaPlayerPart(
startTime: mainStream.startPts.seconds,
endTime: mainStream.startPts.seconds + mainStream.duration,
content: .directFile(ChunkMediaPlayerPart.Content.FFMpegDirectFile(
path: data.path,
audio: segment.audio.flatMap { stream in
return ChunkMediaPlayerPart.DirectStream(
index: stream.index,
startPts: stream.startPts,
endPts: stream.endPts,
duration: stream.duration
)
},
video: segment.video.flatMap { stream in
return ChunkMediaPlayerPart.DirectStream(
index: stream.index,
startPts: stream.startPts,
endPts: stream.endPts,
duration: stream.duration
)
}
)),
codecName: mediaInfo.video?.codecName
))
}
self.partsStateValue.set(.single(ChunkMediaPlayerPartsState(
duration: mainTrack.duration.seconds,
content: .parts(parts)
)))
} else {
self.partsStateValue.set(.single(ChunkMediaPlayerPartsState(
duration: nil,
content: .parts([])
)))
}
} else {
self.partsStateValue.set(.single(ChunkMediaPlayerPartsState(
duration: nil,
content: .parts([])
)))
}
})
}
deinit {
self.completeFetchDisposable?.dispose()
self.dataDisposable?.dispose()
}
func updatePlaybackState(position: Double, isPlaying: Bool) {
func seek(id: Int, position: Double) {
self.seekTimestamp = position
self.currentLookaheadId += 1
let lookaheadId = self.currentLookaheadId
let resource = self.resource
let updateState: (FFMpegLookaheadThread.State) -> Void = { [weak self] state in
Queue.mainQueue().async {
guard let self else {
return
}
if self.currentLookaheadId != lookaheadId {
return
}
guard let mainTrack = state.video ?? state.audio else {
self.partsStateValue.set(.single(ChunkMediaPlayerPartsState(
duration: nil,
content: .directReader(ChunkMediaPlayerPartsState.DirectReader(
id: id,
seekPosition: position,
availableUntilPosition: position,
bufferedUntilEnd: true,
impl: nil
))
)))
return
}
var minAvailableUntilPosition: Double?
if let audio = state.audio {
if let minAvailableUntilPositionValue = minAvailableUntilPosition {
minAvailableUntilPosition = min(minAvailableUntilPositionValue, audio.readableToTime.seconds)
} else {
minAvailableUntilPosition = audio.readableToTime.seconds
}
}
if let video = state.video {
if let minAvailableUntilPositionValue = minAvailableUntilPosition {
minAvailableUntilPosition = min(minAvailableUntilPositionValue, video.readableToTime.seconds)
} else {
minAvailableUntilPosition = video.readableToTime.seconds
}
}
self.partsStateValue.set(.single(ChunkMediaPlayerPartsState(
duration: mainTrack.info.duration.seconds,
content: .directReader(ChunkMediaPlayerPartsState.DirectReader(
id: id,
seekPosition: position,
availableUntilPosition: minAvailableUntilPosition ?? position,
bufferedUntilEnd: state.isEnded,
impl: ChunkMediaPlayerPartsState.DirectReader.Impl(
video: state.video.flatMap { media -> ChunkMediaPlayerPartsState.DirectReader.Stream? in
guard let maxReadablePts = state.maxReadablePts else {
return nil
}
return ChunkMediaPlayerPartsState.DirectReader.Stream(
mediaBox: resource.postbox.mediaBox,
resource: resource.reference.resource,
size: resource.size,
index: media.info.index,
seek: (streamIndex: state.seek.streamIndex, pts: state.seek.pts),
maxReadablePts: (streamIndex: maxReadablePts.streamIndex, pts: maxReadablePts.pts, isEnded: state.isEnded),
codecName: resolveFFMpegCodecName(id: media.info.codecId)
)
},
audio: state.audio.flatMap { media -> ChunkMediaPlayerPartsState.DirectReader.Stream? in
guard let maxReadablePts = state.maxReadablePts else {
return nil
}
return ChunkMediaPlayerPartsState.DirectReader.Stream(
mediaBox: resource.postbox.mediaBox,
resource: resource.reference.resource,
size: resource.size,
index: media.info.index,
seek: (streamIndex: state.seek.streamIndex, pts: state.seek.pts),
maxReadablePts: (streamIndex: maxReadablePts.streamIndex, pts: maxReadablePts.pts, isEnded: state.isEnded),
codecName: resolveFFMpegCodecName(id: media.info.codecId)
)
}
)
))
)))
}
}
self.lookahead = FFMpegLookahead(
seekToTimestamp: position,
updateState: updateState,
fetchInRange: { range in
return fetchedMediaResource(
mediaBox: resource.postbox.mediaBox,
userLocation: resource.userLocation,
userContentType: resource.userContentType,
reference: resource.reference,
range: (range, .elevated),
statsCategory: resource.statsCategory,
preferBackgroundReferenceRevalidation: true
).startStrict()
},
getDataInRange: { range, completion in
return resource.postbox.mediaBox.resourceData(resource.reference.resource, size: resource.size, in: range, mode: .complete).start(next: { result, isComplete in
completion(isComplete ? result : nil)
})
},
isDataCachedInRange: { range in
return resource.postbox.mediaBox.internal_resourceDataIsCached(
id: resource.reference.resource.id,
size: resource.size,
in: range
)
},
size: self.resource.size
)
}
func updatePlaybackState(seekTimestamp: Double, position: Double, isPlaying: Bool) {
if self.seekTimestamp == seekTimestamp {
self.lookahead?.updateCurrentTimestamp(timestamp: position)
}
}
}

536
submodules/MediaPlayer/Sources/ChunkMediaPlayerV2.swift
View File

@ -14,7 +14,8 @@ public let internal_isHardwareAv1Supported: Bool = {
protocol ChunkMediaPlayerSourceImpl: AnyObject {
var partsState: Signal<ChunkMediaPlayerPartsState, NoError> { get }
func updatePlaybackState(position: Double, isPlaying: Bool)
func seek(id: Int, position: Double)
func updatePlaybackState(seekTimestamp: Double, position: Double, isPlaying: Bool)
}
private final class ChunkMediaPlayerExternalSourceImpl: ChunkMediaPlayerSourceImpl {
@ -24,7 +25,10 @@ private final class ChunkMediaPlayerExternalSourceImpl: ChunkMediaPlayerSourceIm
self.partsState = partsState
}
func updatePlaybackState(position: Double, isPlaying: Bool) {
func seek(id: Int, position: Double) {
}
func updatePlaybackState(seekTimestamp: Double, position: Double, isPlaying: Bool) {
}
}
@ -32,14 +36,16 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
public enum SourceDescription {
public final class ResourceDescription {
public let postbox: Postbox
public let size: Int64
public let reference: MediaResourceReference
public let userLocation: MediaResourceUserLocation
public let userContentType: MediaResourceUserContentType
public let statsCategory: MediaResourceStatsCategory
public let fetchAutomatically: Bool
public init(postbox: Postbox, reference: MediaResourceReference, userLocation: MediaResourceUserLocation, userContentType: MediaResourceUserContentType, statsCategory: MediaResourceStatsCategory, fetchAutomatically: Bool) {
public init(postbox: Postbox, size: Int64, reference: MediaResourceReference, userLocation: MediaResourceUserLocation, userContentType: MediaResourceUserContentType, statsCategory: MediaResourceStatsCategory, fetchAutomatically: Bool) {
self.postbox = postbox
self.size = size
self.reference = reference
self.userLocation = userLocation
self.userContentType = userContentType
@ -52,10 +58,23 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
case directFetch(ResourceDescription)
}
public struct MediaDataReaderParams {
public var useV2Reader: Bool
public init(useV2Reader: Bool) {
self.useV2Reader = useV2Reader
}
}
private final class LoadedPart {
enum Content {
case tempFile(ChunkMediaPlayerPart.TempFile)
case directStream(ChunkMediaPlayerPartsState.DirectReader.Stream)
}
final class Media {
let queue: Queue
let content: ChunkMediaPlayerPart.Content
let content: Content
let mediaType: AVMediaType
let codecName: String?
@ -64,7 +83,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
var didBeginReading: Bool = false
var isFinished: Bool = false
init(queue: Queue, content: ChunkMediaPlayerPart.Content, mediaType: AVMediaType, codecName: String?) {
init(queue: Queue, content: Content, mediaType: AVMediaType, codecName: String?) {
assert(queue.isCurrent())
self.queue = queue
@ -77,12 +96,21 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
assert(self.queue.isCurrent())
}
func load() {
func load(params: MediaDataReaderParams) {
let reader: MediaDataReader
if case let .tempFile(tempFile) = self.content, self.mediaType == .video, (self.codecName == "av1" || self.codecName == "av01"), internal_isHardwareAv1Supported {
reader = AVAssetVideoDataReader(filePath: tempFile.file.path, isVideo: self.mediaType == .video)
} else {
reader = FFMpegMediaDataReader(content: self.content, isVideo: self.mediaType == .video, codecName: self.codecName)
switch self.content {
case let .tempFile(tempFile):
if self.mediaType == .video, (self.codecName == "av1" || self.codecName == "av01"), internal_isHardwareAv1Supported {
reader = AVAssetVideoDataReader(filePath: tempFile.file.path, isVideo: self.mediaType == .video)
} else {
if params.useV2Reader {
reader = FFMpegMediaDataReaderV2(content: .tempFile(tempFile), isVideo: self.mediaType == .video, codecName: self.codecName)
} else {
reader = FFMpegMediaDataReaderV1(filePath: tempFile.file.path, isVideo: self.mediaType == .video, codecName: self.codecName)
}
}
case let .directStream(directStream):
reader = FFMpegMediaDataReaderV2(content: .directStream(directStream), isVideo: self.mediaType == .video, codecName: self.codecName)
}
if self.mediaType == .video {
if reader.hasVideo {
@ -94,15 +122,23 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
}
}
}
func update(content: Content) {
if let reader = self.reader {
if let reader = reader as? FFMpegMediaDataReaderV2, case let .directStream(directStream) = content {
reader.update(content: .directStream(directStream))
} else {
assertionFailure()
}
}
}
}
final class MediaData {
let part: ChunkMediaPlayerPart
let video: Media?
let audio: Media?
init(part: ChunkMediaPlayerPart, video: Media?, audio: Media?) {
self.part = part
init(video: Media?, audio: Media?) {
self.video = video
self.audio = audio
}
@ -118,6 +154,8 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
private final class LoadedPartsMediaData {
var ids: [ChunkMediaPlayerPart.Id] = []
var parts: [ChunkMediaPlayerPart.Id: LoadedPart.MediaData] = [:]
var directMediaData: LoadedPart.MediaData?
var directReaderId: Double?
var notifiedHasSound: Bool = false
var seekFromMinTimestamp: Double?
}
@ -125,6 +163,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
private static let sharedDataQueue = Queue(name: "ChunkMediaPlayerV2-DataQueue")
private let dataQueue: Queue
private let mediaDataReaderParams: MediaDataReaderParams
private let audioSessionManager: ManagedAudioSession
private let onSeeked: (() -> Void)?
@ -155,6 +194,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
public var actionAtEnd: MediaPlayerActionAtEnd = .stop
private var didSeekOnce: Bool = false
private var isPlaying: Bool = false
private var baseRate: Double = 1.0
private var isSoundEnabled: Bool
@ -172,6 +212,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
private var audioIsRequestingMediaData: Bool = false
private let source: ChunkMediaPlayerSourceImpl
private var didSetSourceSeek: Bool = false
private var partsStateDisposable: Disposable?
private var updateTimer: Foundation.Timer?
@ -179,6 +220,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
private var hasAudioSession: Bool = false
public init(
params: MediaDataReaderParams,
audioSessionManager: ManagedAudioSession,
source: SourceDescription,
video: Bool,
@ -197,6 +239,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
) {
self.dataQueue = ChunkMediaPlayerV2.sharedDataQueue
self.mediaDataReaderParams = params
self.audioSessionManager = audioSessionManager
self.onSeeked = onSeeked
@ -220,7 +263,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
case let .externalParts(partsState):
self.source = ChunkMediaPlayerExternalSourceImpl(partsState: partsState)
case let .directFetch(resource):
self.source = ChunkMediaPlayerDirectFetchSourceImpl(dataQueue: self.dataQueue, resource: resource)
self.source = ChunkMediaPlayerDirectFetchSourceImpl(resource: resource)
}
self.updateTimer = Foundation.Timer.scheduledTimer(withTimeInterval: 1.0 / 60.0, repeats: true, block: { [weak self] _ in
@ -329,6 +372,12 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
}
}
if !self.didSeekOnce {
self.didSeekOnce = true
self.seek(timestamp: 0.0, play: nil)
return
}
let timestamp: CMTime
if let pendingSeekTimestamp = self.pendingSeekTimestamp {
timestamp = CMTimeMakeWithSeconds(pendingSeekTimestamp, preferredTimescale: 44000)
@ -338,6 +387,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
let timestampSeconds = timestamp.seconds
self.source.updatePlaybackState(
seekTimestamp: self.seekTimestamp,
position: timestampSeconds,
isPlaying: self.isPlaying
)
@ -347,37 +397,16 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
duration = partsStateDuration
}
var validParts: [ChunkMediaPlayerPart] = []
var minStartTime: Double = 0.0
for i in 0 ..< self.partsState.parts.count {
let part = self.partsState.parts[i]
let partStartTime = max(minStartTime, part.startTime)
let partEndTime = max(partStartTime, part.endTime)
if partStartTime >= partEndTime {
continue
}
var partMatches = false
if timestampSeconds >= partStartTime - 0.5 && timestampSeconds < partEndTime + 0.5 {
partMatches = true
}
if partMatches {
validParts.append(ChunkMediaPlayerPart(
startTime: part.startTime,
clippedStartTime: partStartTime == part.startTime ? nil : partStartTime,
endTime: part.endTime,
content: part.content,
codecName: part.codecName
))
minStartTime = max(minStartTime, partEndTime)
}
}
let isBuffering: Bool
if let lastValidPart = validParts.last {
for i in 0 ..< self.partsState.parts.count {
let part = self.partsState.parts[i]
let mediaDataReaderParams = self.mediaDataReaderParams
switch self.partsState.content {
case let .parts(partsStateParts):
var validParts: [ChunkMediaPlayerPart] = []
var minStartTime: Double = 0.0
for i in 0 ..< partsStateParts.count {
let part = partsStateParts[i]
let partStartTime = max(minStartTime, part.startTime)
let partEndTime = max(partStartTime, part.endTime)
@ -385,7 +414,12 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
continue
}
if lastValidPart !== part && partStartTime > (lastValidPart.clippedStartTime ?? lastValidPart.startTime) && partStartTime <= lastValidPart.endTime + 0.5 {
var partMatches = false
if timestampSeconds >= partStartTime - 0.5 && timestampSeconds < partEndTime + 0.5 {
partMatches = true
}
if partMatches {
validParts.append(ChunkMediaPlayerPart(
startTime: part.startTime,
clippedStartTime: partStartTime == part.startTime ? nil : partStartTime,
@ -394,141 +428,247 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
codecName: part.codecName
))
minStartTime = max(minStartTime, partEndTime)
break
}
}
}
if validParts.isEmpty, let pendingContinuePlaybackAfterSeekToTimestamp = self.pendingContinuePlaybackAfterSeekToTimestamp {
for part in self.partsState.parts {
if pendingContinuePlaybackAfterSeekToTimestamp >= part.startTime - 0.2 && pendingContinuePlaybackAfterSeekToTimestamp < part.endTime {
self.renderSynchronizer.setRate(Float(self.renderSynchronizerRate), time: CMTimeMakeWithSeconds(part.startTime, preferredTimescale: 44000))
break
}
}
}
self.loadedParts.removeAll(where: { partState in
if !validParts.contains(where: { $0.id == partState.part.id }) {
return true
}
return false
})
for part in validParts {
if !self.loadedParts.contains(where: { $0.part.id == part.id }) {
self.loadedParts.append(LoadedPart(part: part))
self.loadedParts.sort(by: { $0.part.startTime < $1.part.startTime })
}
}
if self.pendingSeekTimestamp != nil {
return
}
let loadedParts = self.loadedParts
let dataQueue = self.dataQueue
let isSoundEnabled = self.isSoundEnabled
self.loadedPartsMediaData.with { [weak self] loadedPartsMediaData in
loadedPartsMediaData.ids = loadedParts.map(\.part.id)
for part in loadedParts {
if let loadedPart = loadedPartsMediaData.parts[part.part.id] {
if let audio = loadedPart.audio, audio.didBeginReading, !isSoundEnabled {
let cleanAudio = LoadedPart.Media(
if let lastValidPart = validParts.last {
for i in 0 ..< partsStateParts.count {
let part = partsStateParts[i]
let partStartTime = max(minStartTime, part.startTime)
let partEndTime = max(partStartTime, part.endTime)
if partStartTime >= partEndTime {
continue
}
if lastValidPart !== part && partStartTime > (lastValidPart.clippedStartTime ?? lastValidPart.startTime) && partStartTime <= lastValidPart.endTime + 0.5 {
validParts.append(ChunkMediaPlayerPart(
startTime: part.startTime,
clippedStartTime: partStartTime == part.startTime ? nil : partStartTime,
endTime: part.endTime,
content: part.content,
codecName: part.codecName
))
minStartTime = max(minStartTime, partEndTime)
break
}
}
}
if validParts.isEmpty, let pendingContinuePlaybackAfterSeekToTimestamp = self.pendingContinuePlaybackAfterSeekToTimestamp {
for part in partsStateParts {
if pendingContinuePlaybackAfterSeekToTimestamp >= part.startTime - 0.2 && pendingContinuePlaybackAfterSeekToTimestamp < part.endTime {
self.renderSynchronizer.setRate(Float(self.renderSynchronizerRate), time: CMTimeMakeWithSeconds(part.startTime, preferredTimescale: 44000))
break
}
}
}
self.loadedParts.removeAll(where: { partState in
if !validParts.contains(where: { $0.id == partState.part.id }) {
return true
}
return false
})
for part in validParts {
if !self.loadedParts.contains(where: { $0.part.id == part.id }) {
self.loadedParts.append(LoadedPart(part: part))
self.loadedParts.sort(by: { $0.part.startTime < $1.part.startTime })
}
}
var playableDuration: Double = 0.0
var previousValidPartEndTime: Double?
for part in partsStateParts {
if let previousValidPartEndTime {
if part.startTime > previousValidPartEndTime + 0.5 {
break
}
} else if !validParts.contains(where: { $0.id == part.id }) {
continue
}
let partDuration: Double
if part.startTime - 0.5 <= timestampSeconds && part.endTime + 0.5 > timestampSeconds {
partDuration = part.endTime - timestampSeconds
} else if part.startTime - 0.5 > timestampSeconds {
partDuration = part.endTime - part.startTime
} else {
partDuration = 0.0
}
playableDuration += partDuration
previousValidPartEndTime = part.endTime
}
if self.pendingSeekTimestamp != nil {
return
}
let loadedParts = self.loadedParts
let dataQueue = self.dataQueue
let isSoundEnabled = self.isSoundEnabled
self.loadedPartsMediaData.with { [weak self] loadedPartsMediaData in
loadedPartsMediaData.ids = loadedParts.map(\.part.id)
for part in loadedParts {
if let loadedPart = loadedPartsMediaData.parts[part.part.id] {
if let audio = loadedPart.audio, audio.didBeginReading, !isSoundEnabled {
let cleanAudio = LoadedPart.Media(
queue: dataQueue,
content: .tempFile(part.part.content),
mediaType: .audio,
codecName: part.part.codecName
)
cleanAudio.load(params: mediaDataReaderParams)
loadedPartsMediaData.parts[part.part.id] = LoadedPart.MediaData(
video: loadedPart.video,
audio: cleanAudio.reader != nil ? cleanAudio : nil
)
}
} else {
let video = LoadedPart.Media(
queue: dataQueue,
content: part.part.content,
content: .tempFile(part.part.content),
mediaType: .video,
codecName: part.part.codecName
)
video.load(params: mediaDataReaderParams)
let audio = LoadedPart.Media(
queue: dataQueue,
content: .tempFile(part.part.content),
mediaType: .audio,
codecName: part.part.codecName
)
cleanAudio.load()
audio.load(params: mediaDataReaderParams)
loadedPartsMediaData.parts[part.part.id] = LoadedPart.MediaData(
part: part.part,
video: loadedPart.video,
audio: cleanAudio.reader != nil ? cleanAudio : nil
video: video,
audio: audio.reader != nil ? audio : nil
)
}
} else {
let video = LoadedPart.Media(
queue: dataQueue,
content: part.part.content,
mediaType: .video,
codecName: part.part.codecName
)
video.load()
let audio = LoadedPart.Media(
queue: dataQueue,
content: part.part.content,
mediaType: .audio,
codecName: part.part.codecName
)
audio.load()
loadedPartsMediaData.parts[part.part.id] = LoadedPart.MediaData(
part: part.part,
video: video,
audio: audio.reader != nil ? audio : nil
)
}
}
var removedKeys: [ChunkMediaPlayerPart.Id] = []
for (id, _) in loadedPartsMediaData.parts {
if !loadedPartsMediaData.ids.contains(id) {
removedKeys.append(id)
}
}
for id in removedKeys {
loadedPartsMediaData.parts.removeValue(forKey: id)
}
if !loadedPartsMediaData.notifiedHasSound, let part = loadedPartsMediaData.parts.values.first {
loadedPartsMediaData.notifiedHasSound = true
let hasSound = part.audio?.reader != nil
Queue.mainQueue().async {
guard let self else {
return
var removedKeys: [ChunkMediaPlayerPart.Id] = []
for (id, _) in loadedPartsMediaData.parts {
if !loadedPartsMediaData.ids.contains(id) {
removedKeys.append(id)
}
if self.hasSound != hasSound {
self.hasSound = hasSound
self.updateInternalState()
}
for id in removedKeys {
loadedPartsMediaData.parts.removeValue(forKey: id)
}
if !loadedPartsMediaData.notifiedHasSound, let part = loadedPartsMediaData.parts.values.first {
loadedPartsMediaData.notifiedHasSound = true
let hasSound = part.audio?.reader != nil
Queue.mainQueue().async {
guard let self else {
return
}
if self.hasSound != hasSound {
self.hasSound = hasSound
self.updateInternalState()
}
}
}
}
}
var playableDuration: Double = 0.0
var previousValidPartEndTime: Double?
for part in self.partsState.parts {
if let previousValidPartEndTime {
if part.startTime > previousValidPartEndTime + 0.5 {
break
}
} else if !validParts.contains(where: { $0.id == part.id }) {
continue
}
let partDuration: Double
if part.startTime - 0.5 <= timestampSeconds && part.endTime + 0.5 > timestampSeconds {
partDuration = part.endTime - timestampSeconds
} else if part.startTime - 0.5 > timestampSeconds {
partDuration = part.endTime - part.startTime
if let previousValidPartEndTime, previousValidPartEndTime >= duration - 0.5 {
isBuffering = false
} else {
partDuration = 0.0
isBuffering = playableDuration < 1.0
}
case let .directReader(directReader):
var readerImpl: ChunkMediaPlayerPartsState.DirectReader.Impl?
var playableDuration: Double = 0.0
let directReaderSeekPosition = directReader.seekPosition
if directReader.id == self.seekId {
readerImpl = directReader.impl
playableDuration = max(0.0, directReader.availableUntilPosition - timestampSeconds)
if directReader.bufferedUntilEnd {
isBuffering = false
} else {
isBuffering = playableDuration < 1.0
}
} else {
playableDuration = 0.0
isBuffering = true
}
let dataQueue = self.dataQueue
self.loadedPartsMediaData.with { [weak self] loadedPartsMediaData in
if !loadedPartsMediaData.ids.isEmpty {
loadedPartsMediaData.ids = []
}
if !loadedPartsMediaData.parts.isEmpty {
loadedPartsMediaData.parts.removeAll()
}
if let readerImpl {
if let currentDirectMediaData = loadedPartsMediaData.directMediaData, let currentDirectReaderId = loadedPartsMediaData.directReaderId, currentDirectReaderId == directReaderSeekPosition {
if let video = currentDirectMediaData.video, let videoStream = readerImpl.video {
video.update(content: .directStream(videoStream))
}
if let audio = currentDirectMediaData.audio, let audioStream = readerImpl.audio {
audio.update(content: .directStream(audioStream))
}
} else {
let video = readerImpl.video.flatMap { media in
return LoadedPart.Media(
queue: dataQueue,
content: .directStream(media),
mediaType: .video,
codecName: media.codecName
)
}
video?.load(params: mediaDataReaderParams)
let audio = readerImpl.audio.flatMap { media in
return LoadedPart.Media(
queue: dataQueue,
content: .directStream(media),
mediaType: .audio,
codecName: media.codecName
)
}
audio?.load(params: mediaDataReaderParams)
loadedPartsMediaData.directMediaData = LoadedPart.MediaData(
video: video,
audio: audio
)
}
loadedPartsMediaData.directReaderId = directReaderSeekPosition
if !loadedPartsMediaData.notifiedHasSound {
loadedPartsMediaData.notifiedHasSound = true
let hasSound = readerImpl.audio != nil
Queue.mainQueue().async {
guard let self else {
return
}
if self.hasSound != hasSound {
self.hasSound = hasSound
self.updateInternalState()
}
}
}
} else {
loadedPartsMediaData.directMediaData = nil
loadedPartsMediaData.directReaderId = nil
}
}
if self.pendingSeekTimestamp != nil {
return
}
playableDuration += partDuration
previousValidPartEndTime = part.endTime
}
var effectiveRate: Double = 0.0
let isBuffering: Bool
if let previousValidPartEndTime, previousValidPartEndTime >= duration - 0.5 {
isBuffering = false
} else {
isBuffering = playableDuration < 1.0
}
if self.isPlaying {
if !isBuffering {
effectiveRate = self.baseRate
@ -671,6 +811,28 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
}
private func seek(timestamp: Double, play: Bool?, notify: Bool) {
let currentTimestamp: CMTime
if let pendingSeekTimestamp = self.pendingSeekTimestamp {
currentTimestamp = CMTimeMakeWithSeconds(pendingSeekTimestamp, preferredTimescale: 44000)
} else {
currentTimestamp = self.renderSynchronizer.currentTime()
}
let currentTimestampSeconds = currentTimestamp.seconds
if currentTimestampSeconds == timestamp {
if let play {
self.isPlaying = play
}
if notify {
self.shouldNotifySeeked = true
}
if !self.didSetSourceSeek {
self.didSetSourceSeek = true
self.source.seek(id: self.seekId, position: timestamp)
}
self.updateInternalState()
return
}
self.seekId += 1
self.seekTimestamp = timestamp
let seekId = self.seekId
@ -700,6 +862,9 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
audioRenderer.stopRequestingMediaData()
}
self.didSetSourceSeek = true
self.source.seek(id: self.seekId, position: timestamp)
self.loadedPartsMediaData.with { [weak self] loadedPartsMediaData in
loadedPartsMediaData.parts.removeAll()
loadedPartsMediaData.seekFromMinTimestamp = timestamp
@ -734,7 +899,7 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
private func triggerRequestMediaData() {
let loadedPartsMediaData = self.loadedPartsMediaData
if !self.videoIsRequestingMediaData && "".isEmpty {
if !self.videoIsRequestingMediaData {
self.videoIsRequestingMediaData = true
let videoTarget: AVQueuedSampleBufferRendering
@ -805,7 +970,9 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
continue
}
media.didBeginReading = true
if var sampleBuffer = reader.readSampleBuffer() {
switch reader.readSampleBuffer() {
case let .frame(sampleBuffer):
var sampleBuffer = sampleBuffer
if let seekFromMinTimestamp = loadedPartsMediaData.seekFromMinTimestamp, CMSampleBufferGetPresentationTimeStamp(sampleBuffer).seconds < seekFromMinTimestamp {
if isVideo {
var updatedSampleBuffer: CMSampleBuffer?
@ -830,7 +997,54 @@ public final class ChunkMediaPlayerV2: ChunkMediaPlayer {
bufferTarget.enqueue(sampleBuffer)
hasData = true
continue outer
} else {
case .waitingForMoreData, .endOfStream, .error:
media.isFinished = true
}
}
outerDirect: while true {
guard let directMediaData = loadedPartsMediaData.directMediaData else {
break outer
}
guard let media = isVideo ? directMediaData.video : directMediaData.audio else {
break outer
}
if media.isFinished {
break outer
}
guard let reader = media.reader else {
break outer
}
media.didBeginReading = true
switch reader.readSampleBuffer() {
case let .frame(sampleBuffer):
var sampleBuffer = sampleBuffer
if let seekFromMinTimestamp = loadedPartsMediaData.seekFromMinTimestamp, CMSampleBufferGetPresentationTimeStamp(sampleBuffer).seconds < seekFromMinTimestamp {
if isVideo {
var updatedSampleBuffer: CMSampleBuffer?
CMSampleBufferCreateCopy(allocator: nil, sampleBuffer: sampleBuffer, sampleBufferOut: &updatedSampleBuffer)
if let updatedSampleBuffer {
if let attachments = CMSampleBufferGetSampleAttachmentsArray(updatedSampleBuffer, createIfNecessary: true) {
let attachments = attachments as NSArray
let dict = attachments[0] as! NSMutableDictionary
dict.setValue(kCFBooleanTrue as AnyObject, forKey: kCMSampleAttachmentKey_DoNotDisplay as NSString as String)
sampleBuffer = updatedSampleBuffer
}
}
} else {
continue outer
}
}
/*if !isVideo {
print("Enqueue audio \(CMSampleBufferGetPresentationTimeStamp(sampleBuffer).value) next: \(CMSampleBufferGetPresentationTimeStamp(sampleBuffer).value + 1024)")
}*/
bufferTarget.enqueue(sampleBuffer)
hasData = true
continue outer
case .waitingForMoreData:
break outer
case .endOfStream, .error:
media.isFinished = true
}
}

473
submodules/MediaPlayer/Sources/FFMpegFileReader.swift Normal file
View File

@ -0,0 +1,473 @@
import Foundation
#if !os(macOS)
import UIKit
#else
import AppKit
import TGUIKit
#endif
import CoreMedia
import SwiftSignalKit
import FFMpegBinding
import Postbox
import ManagedFile
private func FFMpegFileReader_readPacketCallback(userData: UnsafeMutableRawPointer?, buffer: UnsafeMutablePointer<UInt8>?, bufferSize: Int32) -> Int32 {
guard let buffer else {
return FFMPEG_CONSTANT_AVERROR_EOF
}
let context = Unmanaged<FFMpegFileReader>.fromOpaque(userData!).takeUnretainedValue()
switch context.source {
case let .file(file):
let result = file.read(buffer, Int(bufferSize))
if result == 0 {
return FFMPEG_CONSTANT_AVERROR_EOF
}
return Int32(result)
case let .resource(resource):
let readCount = min(256 * 1024, Int64(bufferSize))
let requestRange: Range<Int64> = resource.readingPosition ..< (resource.readingPosition + readCount)
//TODO:improve thread safe read if incomplete
if let (file, readSize) = resource.mediaBox.internal_resourceData(id: resource.resource.id, size: resource.size, in: requestRange) {
let result = file.read(buffer, readSize)
if result == 0 {
return FFMPEG_CONSTANT_AVERROR_EOF
}
resource.readingPosition += Int64(result)
return Int32(result)
} else {
return FFMPEG_CONSTANT_AVERROR_EOF
}
}
}
private func FFMpegFileReader_seekCallback(userData: UnsafeMutableRawPointer?, offset: Int64, whence: Int32) -> Int64 {
let context = Unmanaged<FFMpegFileReader>.fromOpaque(userData!).takeUnretainedValue()
if (whence & FFMPEG_AVSEEK_SIZE) != 0 {
switch context.source {
case let .file(file):
return file.getSize() ?? 0
case let .resource(resource):
return resource.size
}
} else {
switch context.source {
case let .file(file):
let _ = file.seek(position: offset)
case let .resource(resource):
resource.readingPosition = offset
}
return offset
}
}
final class FFMpegFileReader {
enum SourceDescription {
case file(String)
case resource(mediaBox: MediaBox, resource: MediaResource, size: Int64)
}
final class StreamInfo: Equatable {
let index: Int
let codecId: Int32
let startTime: CMTime
let duration: CMTime
let timeBase: CMTimeValue
let timeScale: CMTimeScale
let fps: CMTime
init(index: Int, codecId: Int32, startTime: CMTime, duration: CMTime, timeBase: CMTimeValue, timeScale: CMTimeScale, fps: CMTime) {
self.index = index
self.codecId = codecId
self.startTime = startTime
self.duration = duration
self.timeBase = timeBase
self.timeScale = timeScale
self.fps = fps
}
static func ==(lhs: StreamInfo, rhs: StreamInfo) -> Bool {
if lhs.index != rhs.index {
return false
}
if lhs.codecId != rhs.codecId {
return false
}
if lhs.startTime != rhs.startTime {
return false
}
if lhs.duration != rhs.duration {
return false
}
if lhs.timeBase != rhs.timeBase {
return false
}
if lhs.timeScale != rhs.timeScale {
return false
}
if lhs.fps != rhs.fps {
return false
}
return true
}
}
fileprivate enum Source {
final class Resource {
let mediaBox: MediaBox
let resource: MediaResource
let size: Int64
var readingPosition: Int64 = 0
init(mediaBox: MediaBox, resource: MediaResource, size: Int64) {
self.mediaBox = mediaBox
self.resource = resource
self.size = size
}
}
case file(ManagedFile)
case resource(Resource)
}
private enum Decoder {
case videoPassthrough(FFMpegMediaPassthroughVideoFrameDecoder)
case video(FFMpegMediaVideoFrameDecoder)
case audio(FFMpegAudioFrameDecoder)
func send(frame: MediaTrackDecodableFrame) -> Bool {
switch self {
case let .videoPassthrough(decoder):
decoder.send(frame: frame)
case let .video(decoder):
decoder.send(frame: frame)
case let .audio(decoder):
decoder.send(frame: frame)
}
}
func sendEnd() -> Bool {
switch self {
case let .videoPassthrough(decoder):
return decoder.sendEndToDecoder()
case let .video(decoder):
return decoder.sendEndToDecoder()
case let .audio(decoder):
return decoder.sendEndToDecoder()
}
}
}
private final class Stream {
let info: StreamInfo
let decoder: Decoder
init(info: StreamInfo, decoder: Decoder) {
self.info = info
self.decoder = decoder
}
}
enum SelectedStream {
enum MediaType {
case audio
case video
}
case mediaType(MediaType)
case index(Int)
}
enum ReadFrameResult {
case frame(MediaTrackFrame)
case waitingForMoreData
case endOfStream
case error
}
private(set) var readingError = false
private var stream: Stream?
private var avIoContext: FFMpegAVIOContext?
private var avFormatContext: FFMpegAVFormatContext?
fileprivate let source: Source
private var didSendEndToDecoder: Bool = false
private var hasReadToEnd: Bool = false
private var maxReadablePts: (streamIndex: Int, pts: Int64, isEnded: Bool)?
private var lastReadPts: (streamIndex: Int, pts: Int64)?
private var isWaitingForMoreData: Bool = false
public init?(source: SourceDescription, passthroughDecoder: Bool = false, useHardwareAcceleration: Bool, selectedStream: SelectedStream, seek: (streamIndex: Int, pts: Int64)?, maxReadablePts: (streamIndex: Int, pts: Int64, isEnded: Bool)?) {
let _ = FFMpegMediaFrameSourceContextHelpers.registerFFMpegGlobals
switch source {
case let .file(path):
guard let file = ManagedFile(queue: nil, path: path, mode: .read) else {
return nil
}
self.source = .file(file)
case let .resource(mediaBox, resource, size):
self.source = .resource(Source.Resource(mediaBox: mediaBox, resource: resource, size: size))
}
self.maxReadablePts = maxReadablePts
let avFormatContext = FFMpegAVFormatContext()
/*if hintVP9 {
avFormatContext.forceVideoCodecId(FFMpegCodecIdVP9)
}*/
let ioBufferSize = 64 * 1024
let avIoContext = FFMpegAVIOContext(bufferSize: Int32(ioBufferSize), opaqueContext: Unmanaged.passUnretained(self).toOpaque(), readPacket: FFMpegFileReader_readPacketCallback, writePacket: nil, seek: FFMpegFileReader_seekCallback, isSeekable: true)
self.avIoContext = avIoContext
avFormatContext.setIO(self.avIoContext!)
if !avFormatContext.openInput(withDirectFilePath: nil) {
self.readingError = true
return nil
}
if !avFormatContext.findStreamInfo() {
self.readingError = true
return nil
}
self.avFormatContext = avFormatContext
var stream: Stream?
outer: for mediaType in [.audio, .video] as [SelectedStream.MediaType] {
streamSearch: for streamIndexNumber in avFormatContext.streamIndices(for: mediaType == .video ? FFMpegAVFormatStreamTypeVideo : FFMpegAVFormatStreamTypeAudio) {
let streamIndex = Int(streamIndexNumber.int32Value)
if avFormatContext.isAttachedPic(atStreamIndex: Int32(streamIndex)) {
continue
}
switch selectedStream {
case let .mediaType(selectedMediaType):
if mediaType != selectedMediaType {
continue streamSearch
}
case let .index(index):
if streamIndex != index {
continue streamSearch
}
}
let codecId = avFormatContext.codecId(atStreamIndex: Int32(streamIndex))
let fpsAndTimebase = avFormatContext.fpsAndTimebase(forStreamIndex: Int32(streamIndex), defaultTimeBase: CMTimeMake(value: 1, timescale: 40000))
let (fps, timebase) = (fpsAndTimebase.fps, fpsAndTimebase.timebase)
let startTime: CMTime
let rawStartTime = avFormatContext.startTime(atStreamIndex: Int32(streamIndex))
if rawStartTime == Int64(bitPattern: 0x8000000000000000 as UInt64) {
startTime = CMTime(value: 0, timescale: timebase.timescale)
} else {
startTime = CMTimeMake(value: rawStartTime, timescale: timebase.timescale)
}
let duration = CMTimeMake(value: avFormatContext.duration(atStreamIndex: Int32(streamIndex)), timescale: timebase.timescale)
let metrics = avFormatContext.metricsForStream(at: Int32(streamIndex))
let rotationAngle: Double = metrics.rotationAngle
//let aspect = Double(metrics.width) / Double(metrics.height)
let info = StreamInfo(
index: streamIndex,
codecId: codecId,
startTime: startTime,
duration: duration,
timeBase: timebase.value,
timeScale: timebase.timescale,
fps: fps
)
switch mediaType {
case .video:
if passthroughDecoder {
var videoFormatData: FFMpegMediaPassthroughVideoFrameDecoder.VideoFormatData?
if codecId == FFMpegCodecIdMPEG4 {
videoFormatData = FFMpegMediaPassthroughVideoFrameDecoder.VideoFormatData(codecType: kCMVideoCodecType_MPEG4Video, width: metrics.width, height: metrics.height, extraData: Data(bytes: metrics.extradata, count: Int(metrics.extradataSize)))
} else if codecId == FFMpegCodecIdH264 {
videoFormatData = FFMpegMediaPassthroughVideoFrameDecoder.VideoFormatData(codecType: kCMVideoCodecType_H264, width: metrics.width, height: metrics.height, extraData: Data(bytes: metrics.extradata, count: Int(metrics.extradataSize)))
} else if codecId == FFMpegCodecIdHEVC {
videoFormatData = FFMpegMediaPassthroughVideoFrameDecoder.VideoFormatData(codecType: kCMVideoCodecType_HEVC, width: metrics.width, height: metrics.height, extraData: Data(bytes: metrics.extradata, count: Int(metrics.extradataSize)))
} else if codecId == FFMpegCodecIdAV1 {
videoFormatData = FFMpegMediaPassthroughVideoFrameDecoder.VideoFormatData(codecType: kCMVideoCodecType_AV1, width: metrics.width, height: metrics.height, extraData: Data(bytes: metrics.extradata, count: Int(metrics.extradataSize)))
}
if let videoFormatData {
stream = Stream(
info: info,
decoder: .videoPassthrough(FFMpegMediaPassthroughVideoFrameDecoder(videoFormatData: videoFormatData, rotationAngle: rotationAngle))
)
break outer
}
} else {
if let codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: useHardwareAcceleration) {
let codecContext = FFMpegAVCodecContext(codec: codec)
if avFormatContext.codecParams(atStreamIndex: Int32(streamIndex), to: codecContext) {
if useHardwareAcceleration {
codecContext.setupHardwareAccelerationIfPossible()
}
if codecContext.open() {
stream = Stream(
info: info,
decoder: .video(FFMpegMediaVideoFrameDecoder(codecContext: codecContext))
)
break outer
}
}
}
}
case .audio:
if let codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: false) {
let codecContext = FFMpegAVCodecContext(codec: codec)
if avFormatContext.codecParams(atStreamIndex: Int32(streamIndex), to: codecContext) {
if codecContext.open() {
stream = Stream(
info: info,
decoder: .audio(FFMpegAudioFrameDecoder(codecContext: codecContext, sampleRate: 48000, channelCount: 1))
)
break outer
}
}
}
}
}
}
guard let stream else {
self.readingError = true
return nil
}
self.stream = stream
if let seek {
avFormatContext.seekFrame(forStreamIndex: Int32(seek.streamIndex), pts: seek.pts, positionOnKeyframe: true)
} else {
avFormatContext.seekFrame(forStreamIndex: Int32(stream.info.index), pts: 0, positionOnKeyframe: true)
}
}
deinit {
}
private func readPacketInternal() -> FFMpegPacket? {
guard let avFormatContext = self.avFormatContext else {
return nil
}
if let maxReadablePts = self.maxReadablePts, !maxReadablePts.isEnded, let lastReadPts = self.lastReadPts, lastReadPts.streamIndex == maxReadablePts.streamIndex, lastReadPts.pts == maxReadablePts.pts {
self.isWaitingForMoreData = true
return nil
}
let packet = FFMpegPacket()
if avFormatContext.readFrame(into: packet) {
self.lastReadPts = (Int(packet.streamIndex), packet.pts)
return packet
} else {
self.hasReadToEnd = true
return nil
}
}
func readDecodableFrame() -> MediaTrackDecodableFrame? {
while !self.readingError && !self.hasReadToEnd && !self.isWaitingForMoreData {
if let packet = self.readPacketInternal() {
if let stream = self.stream, Int(packet.streamIndex) == stream.info.index {
let packetPts = packet.pts
/*if let focusedPart = self.focusedPart, packetPts >= focusedPart.endPts.value {
self.hasReadToEnd = true
}*/
let pts = CMTimeMake(value: packetPts, timescale: stream.info.timeScale)
let dts = CMTimeMake(value: packet.dts, timescale: stream.info.timeScale)
let duration: CMTime
let frameDuration = packet.duration
if frameDuration != 0 {
duration = CMTimeMake(value: frameDuration * stream.info.timeBase, timescale: stream.info.timeScale)
} else {
duration = stream.info.fps
}
let frame = MediaTrackDecodableFrame(type: .video, packet: packet, pts: pts, dts: dts, duration: duration)
return frame
}
} else {
break
}
}
return nil
}
public func readFrame() -> ReadFrameResult {
guard let stream = self.stream else {
return .error
}
while true {
var result: MediaTrackFrame?
switch stream.decoder {
case let .video(decoder):
result = decoder.decode(ptsOffset: nil, forceARGB: false, unpremultiplyAlpha: false, displayImmediately: false)
case let .videoPassthrough(decoder):
result = decoder.decode()
case let .audio(decoder):
result = decoder.decode()
}
if let result {
if self.didSendEndToDecoder {
assert(true)
}
return .frame(result)
}
if !self.isWaitingForMoreData && !self.readingError && !self.hasReadToEnd {
if let decodableFrame = self.readDecodableFrame() {
let _ = stream.decoder.send(frame: decodableFrame)
}
} else if self.hasReadToEnd && !self.didSendEndToDecoder {
self.didSendEndToDecoder = true
let _ = stream.decoder.sendEnd()
} else {
break
}
}
if self.isWaitingForMoreData {
return .waitingForMoreData
} else {
return .endOfStream
}
}
public func updateMaxReadablePts(pts: (streamIndex: Int, pts: Int64, isEnded: Bool)?) {
if self.maxReadablePts?.streamIndex != pts?.streamIndex || self.maxReadablePts?.pts != pts?.pts {
self.maxReadablePts = pts
if let pts {
if pts.isEnded {
self.isWaitingForMoreData = false
} else {
if self.lastReadPts?.streamIndex != pts.streamIndex || self.lastReadPts?.pts != pts.pts {
self.isWaitingForMoreData = false
}
}
} else {
self.isWaitingForMoreData = false
}
}
}
}

4
submodules/MediaPlayer/Sources/FFMpegMediaFrameSourceContext.swift
View File

@ -490,7 +490,7 @@ final class FFMpegMediaFrameSourceContext: NSObject {
let aspect = Double(metrics.width) / Double(metrics.height)
if self.preferSoftwareDecoding {
if let codec = FFMpegAVCodec.find(forId: codecId) {
if let codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: false) {
let codecContext = FFMpegAVCodecContext(codec: codec)
if avFormatContext.codecParams(atStreamIndex: streamIndex, to: codecContext) {
if codecContext.open() {
@ -523,7 +523,7 @@ final class FFMpegMediaFrameSourceContext: NSObject {
var codec: FFMpegAVCodec?
if codec == nil {
codec = FFMpegAVCodec.find(forId: codecId)
codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: false)
}
if let codec = codec {

37
submodules/MediaPlayer/Sources/FFMpegMediaVideoFrameDecoder.swift
View File

@ -277,6 +277,43 @@ public final class FFMpegMediaVideoFrameDecoder: MediaTrackFrameDecoder {
}
private func convertVideoFrame(_ frame: FFMpegAVFrame, pts: CMTime, dts: CMTime, duration: CMTime, forceARGB: Bool = false, unpremultiplyAlpha: Bool = true, displayImmediately: Bool = true) -> MediaTrackFrame? {
if frame.nativePixelFormat() == FFMpegAVFrameNativePixelFormat.videoToolbox {
guard let pixelBufferRef = frame.data[3] else {
return nil
}
let unmanagedPixelBuffer = Unmanaged<CVPixelBuffer>.fromOpaque(UnsafeRawPointer(pixelBufferRef))
let pixelBuffer = unmanagedPixelBuffer.takeUnretainedValue()
var formatRef: CMVideoFormatDescription?
let formatStatus = CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: pixelBuffer, formatDescriptionOut: &formatRef)
guard let format = formatRef, formatStatus == 0 else {
return nil
}
var timingInfo = CMSampleTimingInfo(duration: duration, presentationTimeStamp: pts, decodeTimeStamp: pts)
var sampleBuffer: CMSampleBuffer?
guard CMSampleBufferCreateReadyWithImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: pixelBuffer, formatDescription: format, sampleTiming: &timingInfo, sampleBufferOut: &sampleBuffer) == noErr else {
return nil
}
let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer!, createIfNecessary: true)! as NSArray
let dict = attachments[0] as! NSMutableDictionary
let resetDecoder = self.resetDecoderOnNextFrame
if self.resetDecoderOnNextFrame {
self.resetDecoderOnNextFrame = false
//dict.setValue(kCFBooleanTrue as AnyObject, forKey: kCMSampleBufferAttachmentKey_ResetDecoderBeforeDecoding as NSString as String)
}
if displayImmediately {
dict.setValue(kCFBooleanTrue as AnyObject, forKey: kCMSampleAttachmentKey_DisplayImmediately as NSString as String)
}
return MediaTrackFrame(type: .video, sampleBuffer: sampleBuffer!, resetDecoder: resetDecoder, decoded: true)
}
if frame.data[0] == nil {
return nil
}

184
submodules/MediaPlayer/Sources/MediaDataReader.swift
View File

@ -3,6 +3,7 @@ import AVFoundation
import CoreMedia
import FFMpegBinding
import VideoToolbox
import Postbox
#if os(macOS)
private let internal_isHardwareAv1Supported: Bool = {
@ -11,15 +12,131 @@ private let internal_isHardwareAv1Supported: Bool = {
}()
#endif
public enum MediaDataReaderReadSampleBufferResult {
case frame(CMSampleBuffer)
case waitingForMoreData
case endOfStream
case error
}
public protocol MediaDataReader: AnyObject {
var hasVideo: Bool { get }
var hasAudio: Bool { get }
func readSampleBuffer() -> CMSampleBuffer?
func readSampleBuffer() -> MediaDataReaderReadSampleBufferResult
}
public final class FFMpegMediaDataReader: MediaDataReader {
private let content: ChunkMediaPlayerPart.Content
public final class FFMpegMediaDataReaderV2: MediaDataReader {
public enum Content {
case tempFile(ChunkMediaPlayerPart.TempFile)
case directStream(ChunkMediaPlayerPartsState.DirectReader.Stream)
}
private let content: Content
private let isVideo: Bool
private let videoSource: FFMpegFileReader?
private let audioSource: FFMpegFileReader?
public var hasVideo: Bool {
return self.videoSource != nil
}
public var hasAudio: Bool {
return self.audioSource != nil
}
public init(content: Content, isVideo: Bool, codecName: String?) {
self.content = content
self.isVideo = isVideo
let source: FFMpegFileReader.SourceDescription
var seek: (streamIndex: Int, pts: Int64)?
var maxReadablePts: (streamIndex: Int, pts: Int64, isEnded: Bool)?
switch content {
case let .tempFile(tempFile):
source = .file(tempFile.file.path)
case let .directStream(directStream):
source = .resource(mediaBox: directStream.mediaBox, resource: directStream.resource, size: directStream.size)
seek = (directStream.seek.streamIndex, directStream.seek.pts)
maxReadablePts = directStream.maxReadablePts
}
if self.isVideo {
var passthroughDecoder = true
var useHardwareAcceleration = false
if (codecName == "av1" || codecName == "av01") {
passthroughDecoder = false
useHardwareAcceleration = internal_isHardwareAv1Supported
}
if codecName == "vp9" || codecName == "vp8" {
passthroughDecoder = false
}
/*#if DEBUG
if codecName == "h264" {
passthroughDecoder = false
useHardwareAcceleration = true
}
#endif*/
if let videoSource = FFMpegFileReader(source: source, passthroughDecoder: passthroughDecoder, useHardwareAcceleration: useHardwareAcceleration, selectedStream: .mediaType(.video), seek: seek, maxReadablePts: maxReadablePts) {
self.videoSource = videoSource
} else {
self.videoSource = nil
}
self.audioSource = nil
} else {
if let audioSource = FFMpegFileReader(source: source, passthroughDecoder: false, useHardwareAcceleration: false, selectedStream: .mediaType(.audio), seek: seek, maxReadablePts: maxReadablePts) {
self.audioSource = audioSource
} else {
self.audioSource = nil
}
self.videoSource = nil
}
}
public func update(content: Content) {
guard case let .directStream(directStream) = content else {
return
}
if let audioSource = self.audioSource {
audioSource.updateMaxReadablePts(pts: directStream.maxReadablePts)
} else if let videoSource = self.videoSource {
videoSource.updateMaxReadablePts(pts: directStream.maxReadablePts)
}
}
public func readSampleBuffer() -> MediaDataReaderReadSampleBufferResult {
if let videoSource {
switch videoSource.readFrame() {
case let .frame(frame):
return .frame(frame.sampleBuffer)
case .waitingForMoreData:
return .waitingForMoreData
case .endOfStream:
return .endOfStream
case .error:
return .error
}
} else if let audioSource {
switch audioSource.readFrame() {
case let .frame(frame):
return .frame(frame.sampleBuffer)
case .waitingForMoreData:
return .waitingForMoreData
case .endOfStream:
return .endOfStream
case .error:
return .error
}
} else {
return .endOfStream
}
}
}
public final class FFMpegMediaDataReaderV1: MediaDataReader {
private let isVideo: Bool
private let videoSource: SoftwareVideoReader?
private let audioSource: SoftwareAudioSource?
@ -32,42 +149,15 @@ public final class FFMpegMediaDataReader: MediaDataReader {
return self.audioSource != nil
}
public init(content: ChunkMediaPlayerPart.Content, isVideo: Bool, codecName: String?) {
self.content = content
public init(filePath: String, isVideo: Bool, codecName: String?) {
self.isVideo = isVideo
let filePath: String
var focusedPart: MediaStreamFocusedPart?
switch content {
case let .tempFile(tempFile):
filePath = tempFile.file.path
case let .directFile(directFile):
filePath = directFile.path
let stream = isVideo ? directFile.video : directFile.audio
guard let stream else {
self.videoSource = nil
self.audioSource = nil
return
}
focusedPart = MediaStreamFocusedPart(
seekStreamIndex: stream.index,
startPts: stream.startPts,
endPts: stream.endPts
)
}
if self.isVideo {
var passthroughDecoder = true
if (codecName == "av1" || codecName == "av01") && !internal_isHardwareAv1Supported {
passthroughDecoder = false
}
if codecName == "vp9" || codecName == "vp8" {
passthroughDecoder = false
}
let videoSource = SoftwareVideoReader(path: filePath, hintVP9: false, passthroughDecoder: passthroughDecoder, focusedPart: focusedPart)
let videoSource = SoftwareVideoReader(path: filePath, hintVP9: false, passthroughDecoder: passthroughDecoder)
if videoSource.hasStream {
self.videoSource = videoSource
} else {
@ -75,7 +165,7 @@ public final class FFMpegMediaDataReader: MediaDataReader {
}
self.audioSource = nil
} else {
let audioSource = SoftwareAudioSource(path: filePath, focusedPart: focusedPart)
let audioSource = SoftwareAudioSource(path: filePath)
if audioSource.hasStream {
self.audioSource = audioSource
} else {
@ -85,19 +175,23 @@ public final class FFMpegMediaDataReader: MediaDataReader {
}
}
public func readSampleBuffer() -> CMSampleBuffer? {
public func readSampleBuffer() -> MediaDataReaderReadSampleBufferResult {
if let videoSource {
let frame = videoSource.readFrame()
if let frame {
return frame.sampleBuffer
return .frame(frame.sampleBuffer)
} else {
return nil
return .endOfStream
}
} else if let audioSource {
return audioSource.readSampleBuffer()
if let sampleBuffer = audioSource.readSampleBuffer() {
return .frame(sampleBuffer)
} else {
return .endOfStream
}
} else {
return .endOfStream
}
return nil
}
}
@ -140,14 +234,18 @@ public final class AVAssetVideoDataReader: MediaDataReader {
}
}
public func readSampleBuffer() -> CMSampleBuffer? {
public func readSampleBuffer() -> MediaDataReaderReadSampleBufferResult {
guard let mediaInfo = self.mediaInfo, let assetReader = self.assetReader, let assetOutput = self.assetOutput else {
return nil
return .endOfStream
}
var retryCount = 0
while true {
if let sampleBuffer = assetOutput.copyNextSampleBuffer() {
return createSampleBuffer(fromSampleBuffer: sampleBuffer, withTimeOffset: mediaInfo.startTime, duration: nil)
if let convertedSampleBuffer = createSampleBuffer(fromSampleBuffer: sampleBuffer, withTimeOffset: mediaInfo.startTime, duration: nil) {
return .frame(convertedSampleBuffer)
} else {
return .endOfStream
}
} else if assetReader.status == .reading && retryCount < 100 {
Thread.sleep(forTimeInterval: 1.0 / 60.0)
retryCount += 1
@ -156,7 +254,7 @@ public final class AVAssetVideoDataReader: MediaDataReader {
}
}
return nil
return .endOfStream
}
}

85
submodules/MediaPlayer/Sources/SoftwareVideoSource.swift
View File

@ -163,7 +163,7 @@ public final class SoftwareVideoSource {
break
}
} else {
if let codec = FFMpegAVCodec.find(forId: codecId) {
if let codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: false) {
let codecContext = FFMpegAVCodecContext(codec: codec)
if avFormatContext.codecParams(atStreamIndex: streamIndex, to: codecContext) {
if codecContext.open() {
@ -382,7 +382,6 @@ private func SoftwareAudioSource_seekCallback(userData: UnsafeMutableRawPointer?
}
public final class SoftwareAudioSource {
private let focusedPart: MediaStreamFocusedPart?
private var readingError = false
private var audioStream: SoftwareAudioStream?
private var avIoContext: FFMpegAVIOContext?
@ -397,11 +396,9 @@ public final class SoftwareAudioSource {
return self.audioStream != nil
}
public init(path: String, focusedPart: MediaStreamFocusedPart?) {
public init(path: String) {
let _ = FFMpegMediaFrameSourceContextHelpers.registerFFMpegGlobals
self.focusedPart = focusedPart
var s = stat()
stat(path, &s)
self.size = Int32(s.st_size)
@ -451,7 +448,7 @@ public final class SoftwareAudioSource {
let duration = CMTimeMake(value: avFormatContext.duration(atStreamIndex: streamIndex), timescale: timebase.timescale)
let codec = FFMpegAVCodec.find(forId: codecId)
let codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: false)
if let codec = codec {
let codecContext = FFMpegAVCodecContext(codec: codec)
@ -466,12 +463,8 @@ public final class SoftwareAudioSource {
self.audioStream = audioStream
if let focusedPart = self.focusedPart {
avFormatContext.seekFrame(forStreamIndex: Int32(focusedPart.seekStreamIndex), pts: focusedPart.startPts.value, positionOnKeyframe: true)
} else {
if let audioStream = self.audioStream {
avFormatContext.seekFrame(forStreamIndex: Int32(audioStream.index), pts: 0, positionOnKeyframe: false)
}
if let audioStream = self.audioStream {
avFormatContext.seekFrame(forStreamIndex: Int32(audioStream.index), pts: 0, positionOnKeyframe: false)
}
}
@ -502,10 +495,6 @@ public final class SoftwareAudioSource {
if let audioStream = self.audioStream, Int(packet.streamIndex) == audioStream.index {
let packetPts = packet.pts
if let focusedPart = self.focusedPart, packetPts >= focusedPart.endPts.value {
self.hasReadToEnd = true
}
let pts = CMTimeMake(value: packetPts, timescale: audioStream.timebase.timescale)
let dts = CMTimeMake(value: packet.dts, timescale: audioStream.timebase.timescale)
@ -579,18 +568,6 @@ public final class SoftwareAudioSource {
}
}
public struct MediaStreamFocusedPart {
public let seekStreamIndex: Int
public let startPts: CMTime
public let endPts: CMTime
public init(seekStreamIndex: Int, startPts: CMTime, endPts: CMTime) {
self.seekStreamIndex = seekStreamIndex
self.startPts = startPts
self.endPts = endPts
}
}
private func SoftwareVideoReader_readPacketCallback(userData: UnsafeMutableRawPointer?, buffer: UnsafeMutablePointer<UInt8>?, bufferSize: Int32) -> Int32 {
let context = Unmanaged<SoftwareVideoReader>.fromOpaque(userData!).takeUnretainedValue()
if let fd = context.fd {
@ -617,7 +594,6 @@ private func SoftwareVideoReader_seekCallback(userData: UnsafeMutableRawPointer?
}
final class SoftwareVideoReader {
private let focusedPart: MediaStreamFocusedPart?
private var readingError = false
private var videoStream: SoftwareVideoStream?
private var avIoContext: FFMpegAVIOContext?
@ -636,11 +612,9 @@ final class SoftwareVideoReader {
return self.videoStream != nil
}
public init(path: String, hintVP9: Bool, passthroughDecoder: Bool = false, focusedPart: MediaStreamFocusedPart?) {
public init(path: String, hintVP9: Bool, passthroughDecoder: Bool = false) {
let _ = FFMpegMediaFrameSourceContextHelpers.registerFFMpegGlobals
self.focusedPart = focusedPart
var s = stat()
stat(path, &s)
self.size = Int32(s.st_size)
@ -721,7 +695,7 @@ final class SoftwareVideoReader {
break
}
} else {
if let codec = FFMpegAVCodec.find(forId: codecId) {
if let codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: false) {
let codecContext = FFMpegAVCodecContext(codec: codec)
if avFormatContext.codecParams(atStreamIndex: streamIndex, to: codecContext) {
if codecContext.open() {
@ -737,12 +711,8 @@ final class SoftwareVideoReader {
self.videoStream = videoStream
if let focusedPart = self.focusedPart {
avFormatContext.seekFrame(forStreamIndex: Int32(focusedPart.seekStreamIndex), pts: focusedPart.startPts.value, positionOnKeyframe: true)
} else {
if let videoStream = self.videoStream {
avFormatContext.seekFrame(forStreamIndex: Int32(videoStream.index), pts: 0, positionOnKeyframe: true)
}
if let videoStream = self.videoStream {
avFormatContext.seekFrame(forStreamIndex: Int32(videoStream.index), pts: 0, positionOnKeyframe: true)
}
}
@ -775,10 +745,6 @@ final class SoftwareVideoReader {
if let videoStream = self.videoStream, Int(packet.streamIndex) == videoStream.index {
let packetPts = packet.pts
if let focusedPart = self.focusedPart, packetPts >= focusedPart.endPts.value {
self.hasReadToEnd = true
}
let pts = CMTimeMake(value: packetPts, timescale: videoStream.timebase.timescale)
let dts = CMTimeMake(value: packet.dts, timescale: videoStream.timebase.timescale)
@ -958,21 +924,8 @@ public func extractFFMpegMediaInfo(path: String) -> FFMpegMediaInfo? {
var duration = CMTimeMake(value: avFormatContext.duration(atStreamIndex: streamIndex), timescale: timebase.timescale)
duration = CMTimeMaximum(CMTime(value: 0, timescale: duration.timescale), CMTimeSubtract(duration, startTime))
var codecName: String?
let codecId = avFormatContext.codecId(atStreamIndex: streamIndex)
if codecId == FFMpegCodecIdMPEG4 {
codecName = "mpeg4"
} else if codecId == FFMpegCodecIdH264 {
codecName = "h264"
} else if codecId == FFMpegCodecIdHEVC {
codecName = "hevc"
} else if codecId == FFMpegCodecIdAV1 {
codecName = "av1"
} else if codecId == FFMpegCodecIdVP9 {
codecName = "vp9"
} else if codecId == FFMpegCodecIdVP8 {
codecName = "vp8"
}
let codecName = resolveFFMpegCodecName(id: codecId)
streamInfos.append((isVideo: isVideo, info: FFMpegMediaInfo.Info(
index: Int(streamIndex),
@ -987,3 +940,21 @@ public func extractFFMpegMediaInfo(path: String) -> FFMpegMediaInfo? {
return FFMpegMediaInfo(audio: streamInfos.first(where: { !$0.isVideo })?.info, video: streamInfos.first(where: { $0.isVideo })?.info)
}
public func resolveFFMpegCodecName(id: Int32) -> String? {
if id == FFMpegCodecIdMPEG4 {
return "mpeg4"
} else if id == FFMpegCodecIdH264 {
return "h264"
} else if id == FFMpegCodecIdHEVC {
return "hevc"
} else if id == FFMpegCodecIdAV1 {
return "av1"
} else if id == FFMpegCodecIdVP9 {
return "vp9"
} else if id == FFMpegCodecIdVP8 {
return "vp8"
} else {
return nil
}
}

3
submodules/MediaPlayer/Sources/UniversalSoftwareVideoSource.swift
View File

@ -146,7 +146,6 @@ private final class UniversalSoftwareVideoSourceImpl {
self.size = sizeValue
}
self.mediaBox = mediaBox
self.source = source
self.automaticallyFetchHeader = automaticallyFetchHeader
@ -210,7 +209,7 @@ private final class UniversalSoftwareVideoSourceImpl {
let rotationAngle: Double = metrics.rotationAngle
let aspect = Double(metrics.width) / Double(metrics.height)
if let codec = FFMpegAVCodec.find(forId: codecId) {
if let codec = FFMpegAVCodec.find(forId: codecId, preferHardwareAccelerationCapable: false) {
let codecContext = FFMpegAVCodecContext(codec: codec)
if avFormatContext.codecParams(atStreamIndex: streamIndex, to: codecContext) {
if codecContext.open() {

11
submodules/Postbox/Sources/MediaBox.swift
View File

@ -705,6 +705,17 @@ public final class MediaBox {
}
}
public func internal_resourceDataIsCached(id: MediaResourceId, size: Int64, in range: Range<Int64>) -> Bool {
let paths = self.storePathsForId(id)
if let _ = fileSize(paths.complete) {
return true
} else {
let tempManager = MediaBoxFileManager(queue: nil)
return MediaBoxPartialFile.internal_isPartialDataCached(manager: tempManager, path: paths.partial, metaPath: paths.partial + ".meta", range: range)
}
}
public func resourceData(id: MediaResourceId, size: Int64, in range: Range<Int64>, mode: ResourceDataRangeMode = .complete, notifyAboutIncomplete: Bool = false, attemptSynchronously: Bool = false) -> Signal<(Data, Bool), NoError> {
return Signal { subscriber in
let disposable = MetaDisposable()

10
submodules/Postbox/Sources/MediaBoxFile.swift
View File

@ -106,6 +106,16 @@ final class MediaBoxPartialFile {
return (fd, Int(clippedRange.upperBound - clippedRange.lowerBound))
}
static func internal_isPartialDataCached(manager: MediaBoxFileManager, path: String, metaPath: String, range: Range<Int64>) -> Bool {
guard let fileMap = try? MediaBoxFileMap.read(manager: manager, path: metaPath) else {
return false
}
guard let _ = fileMap.contains(range) else {
return false
}
return true
}
var storedSize: Int64 {
assert(self.queue.isCurrent())
return self.fileMap.sum

6
submodules/Postbox/Sources/TimeBasedCleanup.swift
View File

@ -513,7 +513,11 @@ private final class TimeBasedCleanupImpl {
}
func touch(paths: [String]) {
self.scheduledTouches.append(contentsOf: paths)
for path in paths {
if !self.scheduledTouches.contains(path) {
self.scheduledTouches.append(path)
}
}
self.scheduleTouches()
}

20
submodules/TelegramCallsUI/Sources/PresentationGroupCall.swift
View File

@ -2365,22 +2365,28 @@ public final class PresentationGroupCallImpl: PresentationGroupCall {
if let muteState = filteredMuteState {
if muteState.canUnmute {
switch strongSelf.isMutedValue {
case let .muted(isPushToTalkActive):
if !isPushToTalkActive {
strongSelf.genericCallContext?.setIsMuted(true)
}
case .unmuted:
if let currentMuteState = strongSelf.stateValue.muteState, !currentMuteState.canUnmute {
strongSelf.isMutedValue = .muted(isPushToTalkActive: false)
strongSelf.isMutedPromise.set(strongSelf.isMutedValue)
strongSelf.stateValue.muteState = GroupCallParticipantsContext.Participant.MuteState(canUnmute: true, mutedByYou: false)
strongSelf.genericCallContext?.setIsMuted(true)
} else {
switch strongSelf.isMutedValue {
case .muted:
break
case .unmuted:
let _ = strongSelf.updateMuteState(peerId: strongSelf.joinAsPeerId, isMuted: false)
}
}
} else {
strongSelf.isMutedValue = .muted(isPushToTalkActive: false)
strongSelf.isMutedPromise.set(strongSelf.isMutedValue)
strongSelf.genericCallContext?.setIsMuted(true)
strongSelf.stateValue.muteState = muteState
}
strongSelf.stateValue.muteState = muteState
} else if let currentMuteState = strongSelf.stateValue.muteState, !currentMuteState.canUnmute {
strongSelf.isMutedValue = .muted(isPushToTalkActive: false)
strongSelf.isMutedPromise.set(strongSelf.isMutedValue)
strongSelf.stateValue.muteState = GroupCallParticipantsContext.Participant.MuteState(canUnmute: true, mutedByYou: false)
strongSelf.genericCallContext?.setIsMuted(true)
}

8
submodules/TelegramUniversalVideoContent/Sources/HLSVideoContent.swift
View File

@ -27,7 +27,7 @@ public struct HLSCodecConfiguration {
public extension HLSCodecConfiguration {
init(context: AccountContext) {
/*var isSoftwareAv1Supported = false
var isSoftwareAv1Supported = false
var isHardwareAv1Supported = false
var length: Int = 4
@ -44,9 +44,7 @@ public extension HLSCodecConfiguration {
isSoftwareAv1Supported = value != 0.0
}
self.init(isHardwareAv1Supported: isHardwareAv1Supported, isSoftwareAv1Supported: isSoftwareAv1Supported)*/
self.init(isHardwareAv1Supported: false, isSoftwareAv1Supported: false)
self.init(isHardwareAv1Supported: isHardwareAv1Supported, isSoftwareAv1Supported: isSoftwareAv1Supported)
}
}
@ -265,7 +263,7 @@ public final class HLSVideoContent: UniversalVideoContent {
}
public func makeContentNode(context: AccountContext, postbox: Postbox, audioSession: ManagedAudioSession) -> UniversalVideoContentNode & ASDisplayNode {
return HLSVideoJSNativeContentNode(accountId: context.account.id, postbox: postbox, audioSessionManager: audioSession, userLocation: self.userLocation, fileReference: self.fileReference, streamVideo: self.streamVideo, loopVideo: self.loopVideo, enableSound: self.enableSound, baseRate: self.baseRate, fetchAutomatically: self.fetchAutomatically, onlyFullSizeThumbnail: self.onlyFullSizeThumbnail, useLargeThumbnail: self.useLargeThumbnail, autoFetchFullSizeThumbnail: self.autoFetchFullSizeThumbnail, codecConfiguration: self.codecConfiguration)
return HLSVideoJSNativeContentNode(context: context, postbox: postbox, audioSessionManager: audioSession, userLocation: self.userLocation, fileReference: self.fileReference, streamVideo: self.streamVideo, loopVideo: self.loopVideo, enableSound: self.enableSound, baseRate: self.baseRate, fetchAutomatically: self.fetchAutomatically, onlyFullSizeThumbnail: self.onlyFullSizeThumbnail, useLargeThumbnail: self.useLargeThumbnail, autoFetchFullSizeThumbnail: self.autoFetchFullSizeThumbnail, codecConfiguration: self.codecConfiguration)
}
public func isEqual(to other: UniversalVideoContent) -> Bool {

11
submodules/TelegramUniversalVideoContent/Sources/HLSVideoJSNativeContentNode.swift
View File

@ -1029,7 +1029,7 @@ final class HLSVideoJSNativeContentNode: ASDisplayNode, UniversalVideoContentNod
private var contextDisposable: Disposable?
init(accountId: AccountRecordId, postbox: Postbox, audioSessionManager: ManagedAudioSession, userLocation: MediaResourceUserLocation, fileReference: FileMediaReference, streamVideo: Bool, loopVideo: Bool, enableSound: Bool, baseRate: Double, fetchAutomatically: Bool, onlyFullSizeThumbnail: Bool, useLargeThumbnail: Bool, autoFetchFullSizeThumbnail: Bool, codecConfiguration: HLSCodecConfiguration) {
init(context: AccountContext, postbox: Postbox, audioSessionManager: ManagedAudioSession, userLocation: MediaResourceUserLocation, fileReference: FileMediaReference, streamVideo: Bool, loopVideo: Bool, enableSound: Bool, baseRate: Double, fetchAutomatically: Bool, onlyFullSizeThumbnail: Bool, useLargeThumbnail: Bool, autoFetchFullSizeThumbnail: Bool, codecConfiguration: HLSCodecConfiguration) {
self.instanceId = HLSVideoJSNativeContentNode.nextInstanceId
HLSVideoJSNativeContentNode.nextInstanceId += 1
@ -1059,7 +1059,7 @@ final class HLSVideoJSNativeContentNode: ASDisplayNode, UniversalVideoContentNod
var playerSource: HLSJSServerSource?
if let qualitySet = HLSQualitySet(baseFile: fileReference, codecConfiguration: codecConfiguration) {
let playerSourceValue = HLSJSServerSource(accountId: accountId.int64, fileId: fileReference.media.fileId.id, postbox: postbox, userLocation: userLocation, playlistFiles: qualitySet.playlistFiles, qualityFiles: qualitySet.qualityFiles)
let playerSourceValue = HLSJSServerSource(accountId: context.account.id.int64, fileId: fileReference.media.fileId.id, postbox: postbox, userLocation: userLocation, playlistFiles: qualitySet.playlistFiles, qualityFiles: qualitySet.qualityFiles)
playerSource = playerSourceValue
}
self.playerSource = playerSource
@ -1075,6 +1075,7 @@ final class HLSVideoJSNativeContentNode: ASDisplayNode, UniversalVideoContentNod
var onSeeked: (() -> Void)?
self.player = ChunkMediaPlayerV2(
params: ChunkMediaPlayerV2.MediaDataReaderParams(context: context),
audioSessionManager: audioSessionManager,
source: .externalParts(self.chunkPlayerPartsState.get()),
video: true,
@ -1102,9 +1103,9 @@ final class HLSVideoJSNativeContentNode: ASDisplayNode, UniversalVideoContentNod
self.playerNode.isHidden = false
}*/
//let thumbnailVideoReference = HLSVideoContent.minimizedHLSQuality(file: fileReference)?.file ?? fileReference
let thumbnailVideoReference = HLSVideoContent.minimizedHLSQuality(file: fileReference, codecConfiguration: self.codecConfiguration)?.file ?? fileReference
self.imageNode.setSignal(internalMediaGridMessageVideo(postbox: postbox, userLocation: userLocation, videoReference: fileReference, previewSourceFileReference: nil, imageReference: nil, onlyFullSize: onlyFullSizeThumbnail, useLargeThumbnail: useLargeThumbnail, autoFetchFullSizeThumbnail: autoFetchFullSizeThumbnail || fileReference.media.isInstantVideo) |> map { [weak self] getSize, getData in
self.imageNode.setSignal(internalMediaGridMessageVideo(postbox: postbox, userLocation: userLocation, videoReference: thumbnailVideoReference, previewSourceFileReference: nil, imageReference: nil, onlyFullSize: onlyFullSizeThumbnail, useLargeThumbnail: useLargeThumbnail, autoFetchFullSizeThumbnail: autoFetchFullSizeThumbnail || fileReference.media.isInstantVideo) |> map { [weak self] getSize, getData in
Queue.mainQueue().async {
if let strongSelf = self, strongSelf.dimensions == nil {
if let dimensions = getSize() {
@ -1832,7 +1833,7 @@ private final class SourceBuffer {
let item = ChunkMediaPlayerPart(
startTime: fragmentInfo.startTime.seconds,
endTime: fragmentInfo.startTime.seconds + fragmentInfo.duration.seconds,
content: .tempFile(ChunkMediaPlayerPart.Content.TempFile(file: tempFile)),
content: ChunkMediaPlayerPart.TempFile(file: tempFile),
codecName: videoCodecName
)
self.items.append(item)

13
submodules/TelegramUniversalVideoContent/Sources/NativeVideoContent.swift
View File

@ -277,6 +277,17 @@ private enum PlayerImpl {
}
}
extension ChunkMediaPlayerV2.MediaDataReaderParams {
init(context: AccountContext) {
var useV2Reader = true
if let data = context.currentAppConfiguration.with({ $0 }).data, let value = data["ios_video_v2_reader"] as? Double {
useV2Reader = value != 0.0
}
self.init(useV2Reader: useV2Reader)
}
}
private final class NativeVideoContentNode: ASDisplayNode, UniversalVideoContentNode {
private let postbox: Postbox
private let userLocation: MediaResourceUserLocation
@ -510,9 +521,11 @@ private final class NativeVideoContentNode: ASDisplayNode, UniversalVideoContent
self.initializePlayer(player: .legacy(mediaPlayer))
} else {
let mediaPlayer = ChunkMediaPlayerV2(
params: ChunkMediaPlayerV2.MediaDataReaderParams(context: context),
audioSessionManager: audioSessionManager,
source: .directFetch(ChunkMediaPlayerV2.SourceDescription.ResourceDescription(
postbox: postbox,
size: selectedFile.size ?? 0,
reference: fileReference.resourceReference(selectedFile.resource),
userLocation: userLocation,
userContentType: userContentType,