ThePain/Swiftgram
1
0
Fork 0
mirror of https://github.com/Swiftgram/Telegram-iOS.git synced 2025-12-23 14:45:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Camera and editor improvements

Browse Source
This commit is contained in:
Ilya Laktyushin
2023-05-26 17:32:02 +04:00
parent d086a8f674
commit eeb1c469f3
64 changed files with 4037 additions and 1108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
submodules/TelegramUI/Components/MediaEditor/Sources/VideoTextureSource.swift
View File

@@ -13,22 +13,19 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
private var displayLink: CADisplayLink?
private let device: MTLDevice?
private var textureRotation: TextureRotation = .rotate0Degrees
private var forceUpdate: Bool = false
weak var output: TextureConsumer?
var textureCache: CVMetalTextureCache!
var queue: DispatchQueue!
var started: Bool = false
init(player: AVPlayer, renderTarget: RenderTarget) {
self.player = player
if let device = renderTarget.mtlDevice, CVMetalTextureCacheCreate(nil, nil, device, nil, &self.textureCache) != kCVReturnSuccess {
print("error")
}
self.device = renderTarget.mtlDevice!
self.queue = DispatchQueue(
label: "VideoTextureSource Queue",
qos: .userInteractive,
@@ -47,7 +44,8 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
}
deinit {
print()
self.playerItemObservation?.invalidate()
self.playerItemStatusObservation?.invalidate()
}
private func updatePlayerItem(_ playerItem: AVPlayerItem?) {
@@ -63,7 +61,7 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
self.playerItemStatusObservation = nil
self.playerItem = playerItem
self.playerItemStatusObservation = self.playerItem?.observe(\.status, options: [.initial,.new], changeHandler: { [weak self] item, change in
self.playerItemStatusObservation = self.playerItem?.observe(\.status, options: [.initial, .new], changeHandler: { [weak self] item, change in
guard let strongSelf = self else {
return
}
@@ -91,21 +89,10 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
} else if t.b == -1.0 && t.c == 1.0 {
self.textureRotation = .rotate270Degrees
} else if t.a == -1.0 && t.d == 1.0 {
// if (mirrored != NULL) {
// *mirrored = true;
// }
self.textureRotation = .rotate270Degrees
} else if t.a == 1.0 && t.d == -1.0 {
// if (mirrored != NULL) {
// *mirrored = true;
// }
self.textureRotation = .rotate180Degrees
} else {
// if (t.c == 1) {
// if (mirrored != NULL) {
// *mirrored = true;
// }
// }
self.textureRotation = .rotate90Degrees
}
}
@@ -115,7 +102,20 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
return
}
let output = AVPlayerItemVideoOutput(pixelBufferAttributes: [kCVPixelBufferPixelFormatTypeKey as NSString as String: kCVPixelFormatType_32BGRA])
let colorProperties: [String: Any] = [
AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2,
AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2,
AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2
]
let outputSettings: [String: Any] = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
kCVPixelBufferMetalCompatibilityKey as String: true,
AVVideoColorPropertiesKey: colorProperties
]
let output = AVPlayerItemVideoOutput(outputSettings: outputSettings)
output.suppressesPlayerRendering = true
output.setDelegate(self, queue: self.queue)
playerItem.add(output)
self.playerItemOutput = output
@@ -174,12 +174,10 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
var presentationTime: CMTime = .zero
if let pixelBuffer = output.copyPixelBuffer(forItemTime: requestTime, itemTimeForDisplay: &presentationTime) {
if let texture = self.pixelBufferToMTLTexture(pixelBuffer: pixelBuffer) {
self.output?.consumeTexture(texture, rotation: self.textureRotation)
}
self.output?.consumeVideoPixelBuffer(pixelBuffer, rotation: self.textureRotation)
}
}
func setNeedsUpdate() {
self.displayLink?.isPaused = false
self.forceUpdate = true
@@ -196,19 +194,88 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
self.output = consumer
}
private func pixelBufferToMTLTexture(pixelBuffer: CVPixelBuffer) -> MTLTexture? {
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let format: MTLPixelFormat = .bgra8Unorm
var textureRef : CVMetalTexture?
let status = CVMetalTextureCacheCreateTextureFromImage(nil, self.textureCache, pixelBuffer, nil, format, width, height, 0, &textureRef)
if status == kCVReturnSuccess {
return CVMetalTextureGetTexture(textureRef!)
}
return nil
}
public func outputMediaDataWillChange(_ sender: AVPlayerItemOutput) {
self.displayLink?.isPaused = false
}
}
final class VideoInputPass: DefaultRenderPass {
private var cachedTexture: MTLTexture?
override var fragmentShaderFunctionName: String {
return "bt709ToRGBFragmentShader"
}
func processPixelBuffer(_ pixelBuffer: CVPixelBuffer, rotation: TextureRotation, textureCache: CVMetalTextureCache, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
func textureFromPixelBuffer(_ pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, width: Int, height: Int, plane: Int) -> MTLTexture? {
var textureRef : CVMetalTexture?
let status = CVMetalTextureCacheCreateTextureFromImage(nil, textureCache, pixelBuffer, nil, pixelFormat, width, height, plane, &textureRef)
if status == kCVReturnSuccess, let textureRef {
return CVMetalTextureGetTexture(textureRef)
}
return nil
}
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
guard let inputYTexture = textureFromPixelBuffer(pixelBuffer, pixelFormat: .r8Unorm, width: width, height: height, plane: 0),
let inputCbCrTexture = textureFromPixelBuffer(pixelBuffer, pixelFormat: .rg8Unorm, width: width >> 1, height: height >> 1, plane: 1) else {
return nil
}
return self.process(yTexture: inputYTexture, cbcrTexture: inputCbCrTexture, width: width, height: height, rotation: rotation, device: device, commandBuffer: commandBuffer)
}
func process(yTexture: MTLTexture, cbcrTexture: MTLTexture, width: Int, height: Int, rotation: TextureRotation, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device, rotation: rotation)
func textureDimensionsForRotation(width: Int, height: Int, rotation: TextureRotation) -> (width: Int, height: Int) {
switch rotation {
case .rotate90Degrees, .rotate270Degrees:
return (height, width)
default:
return (width, height)
}
}
let (outputWidth, outputHeight) = textureDimensionsForRotation(width: width, height: height, rotation: rotation)
// let outputSize = CGSize(width: outputWidth, height: outputHeight).fitted(CGSize(width: 1920.0, height: 1920.0))
// outputWidth = Int(outputSize.width)
// outputHeight = Int(outputSize.height)
if self.cachedTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = outputWidth
textureDescriptor.height = outputHeight
textureDescriptor.pixelFormat = .bgra8Unorm
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
if let texture = device.makeTexture(descriptor: textureDescriptor) {
self.cachedTexture = texture
}
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return nil
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(outputWidth), height: Double(outputHeight),
znear: -1.0, zfar: 1.0)
)
renderCommandEncoder.setFragmentTexture(yTexture, index: 0)
renderCommandEncoder.setFragmentTexture(cbcrTexture, index: 1)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return self.cachedTexture
}
}