Camera and editor improvements

submodules/TelegramUI/Components/MediaEditor/MetalResources/EditorDefault.metal

@@ -21,34 +21,17 @@ vertex RasterizerData defaultVertexShader(uint vertexID [[vertex_id]],
 }
 
 fragment half4 defaultFragmentShader(RasterizerData in [[stage_in]],
-                                      constant float2 &texCoordScales [[buffer(0)]],
                                       texture2d<half, access::sample> texture [[texture(0)]]) {
     constexpr sampler samplr(filter::linear, mag_filter::linear, min_filter::linear);
-    
+    half3 color = texture.sample(samplr, in.texCoord).rgb;
-    float scaleX = texCoordScales.x;
-    float scaleY = texCoordScales.y;
-    float x = (in.texCoord.x - (1.0 - scaleX) / 2.0) / scaleX;
-    float y = (in.texCoord.y - (1.0 - scaleY) / 2.0) / scaleY;
-    if (x < 0 || x > 1 || y < 0 || y > 1) {
-        return half4(0.0, 0.0, 0.0, 1.0);
-    }
-    half3 color = texture.sample(samplr, float2(x, y)).rgb;
     return half4(color, 1.0);
 }
 
 fragment half histogramPrepareFragmentShader(RasterizerData in [[stage_in]],
-                                              constant float2 &texCoordScales [[buffer(0)]],
                                               texture2d<half, access::sample> texture [[texture(0)]]) {
     constexpr sampler samplr(filter::linear, mag_filter::linear, min_filter::linear);
     
-    float scaleX = texCoordScales.x;
+    half3 color = texture.sample(samplr, in.texCoord).rgb;
-    float scaleY = texCoordScales.y;
-    float x = (in.texCoord.x - (1.0 - scaleX) / 2.0) / scaleX;
-    float y = (in.texCoord.y - (1.0 - scaleY) / 2.0) / scaleY;
-    if (x < 0 || x > 1 || y < 0 || y > 1) {
-        return 0.0;
-    }
-    half3 color = texture.sample(samplr, float2(x, y)).rgb;
     half luma = color.r * 0.3 + color.g * 0.59 + color.b * 0.11;
     return luma;
 }

submodules/TelegramUI/Components/MediaEditor/Sources/HistogramCalculationPass.swift

@@ -79,10 +79,7 @@ final class HistogramCalculationPass: DefaultRenderPass {
             )
             
             renderCommandEncoder.setFragmentTexture(input, index: 0)
-            
+
-            var texCoordScales = simd_float2(x: 1.0, y: 1.0)
-            renderCommandEncoder.setFragmentBytes(&texCoordScales, length: MemoryLayout<simd_float2>.stride, index: 0)
-            
             self.encodeDefaultCommands(using: renderCommandEncoder)
             
             renderCommandEncoder.endEncoding()

submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorComposer.swift

@@ -56,7 +56,6 @@ final class MediaEditorComposer {
         self.colorSpace = colorSpace
         
         self.renderer.addRenderChain(self.renderChain)
-        self.renderer.addRenderPass(ComposerRenderPass())
         
         if let gradientColors = values.gradientColors, let image = mediaEditorGenerateGradientImage(size: dimensions, colors: gradientColors) {
             self.gradientImage = CIImage(image: image, options: [.colorSpace: self.colorSpace])!.transformed(by: CGAffineTransform(translationX: -dimensions.width / 2.0, y: -dimensions.height / 2.0))
@@ -106,7 +105,7 @@ final class MediaEditorComposer {
                     if var compositedImage {
                         let scale = self.outputDimensions.width / self.dimensions.width
                         compositedImage = compositedImage.transformed(by: CGAffineTransform(scaleX: scale, y: scale))
-                        
+ 
                         self.ciContext?.render(compositedImage, to: pixelBuffer)
                         completion(pixelBuffer)
                     } else {
@@ -568,55 +567,3 @@ private func render(width: Int, height: Int, bytesPerRow: Int, data: Data, type:
     
     return CIImage(cvPixelBuffer: pixelBuffer, options: [.colorSpace: colorSpace])
 }
-
-final class ComposerRenderPass: DefaultRenderPass {
-    fileprivate var cachedTexture: MTLTexture?
-    
-    override func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
-        self.setupVerticesBuffer(device: device)
-        
-        let width = input.width
-        let height = input.height
-        
-        if self.cachedTexture == nil || self.cachedTexture?.width != width || self.cachedTexture?.height != height {
-            let textureDescriptor = MTLTextureDescriptor()
-            textureDescriptor.textureType = .type2D
-            textureDescriptor.width = width
-            textureDescriptor.height = height
-            textureDescriptor.pixelFormat = input.pixelFormat
-            textureDescriptor.storageMode = .shared
-            textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
-            guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
-                return input
-            }
-            self.cachedTexture = texture
-            texture.label = "composerTexture"
-        }
-        
-        let renderPassDescriptor = MTLRenderPassDescriptor()
-        renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
-        renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
-        renderPassDescriptor.colorAttachments[0].storeAction = .store
-        renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 1.0)
-        guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
-            return input
-        }
-        
-        renderCommandEncoder.setViewport(MTLViewport(
-            originX: 0, originY: 0,
-            width: Double(width), height: Double(height),
-            znear: -1.0, zfar: 1.0)
-        )
-        
-        renderCommandEncoder.setFragmentTexture(input, index: 0)
-        
-        var texCoordScales = simd_float2(x: 1.0, y: 1.0)
-        renderCommandEncoder.setFragmentBytes(&texCoordScales, length: MemoryLayout<simd_float2>.stride, index: 0)
-        
-        self.encodeDefaultCommands(using: renderCommandEncoder)
-        
-        renderCommandEncoder.endEncoding()
-        
-        return self.cachedTexture!
-    }
-}

submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoExport.swift

@@ -326,7 +326,7 @@ public final class MediaEditorVideoExport {
         if (videoTracks.count > 0) {
             var sourceFrameRate: Float = 0.0
             let outputSettings: [String: Any]  = [
-                kCVPixelBufferPixelFormatTypeKey as String: [kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange],
+                kCVPixelBufferPixelFormatTypeKey as String: [kCVPixelFormatType_420YpCbCr8BiPlanarFullRange],
                 AVVideoColorPropertiesKey: [
                     AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2,
                     AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2,

submodules/TelegramUI/Components/MediaEditor/Sources/RenderPass.swift

@@ -147,23 +147,8 @@ final class OutputRenderPass: DefaultRenderPass {
             width: Double(drawableSize.width), height: Double(drawableSize.height),
             znear: -1.0, zfar: 1.0))
         
-        do {
+
-            var texCoordScales = simd_float2(x: 1.0, y: 1.0)
+        renderCommandEncoder.setFragmentTexture(input, index: 0)
-            var scaleFactor = drawableSize.width / CGFloat(input.width)
-            let textureFitHeight = CGFloat(input.height) * scaleFactor
-            if textureFitHeight > drawableSize.height {
-                scaleFactor = drawableSize.height / CGFloat(input.height)
-                let textureFitWidth = CGFloat(input.width) * scaleFactor
-                let texCoordsScaleX = textureFitWidth / drawableSize.width
-                texCoordScales.x = Float(texCoordsScaleX)
-            } else {
-                let texCoordsScaleY = textureFitHeight / drawableSize.height
-                texCoordScales.y = Float(texCoordsScaleY)
-            }
-            
-            renderCommandEncoder.setFragmentBytes(&texCoordScales, length: MemoryLayout<simd_float2>.stride, index: 0)
-            renderCommandEncoder.setFragmentTexture(input, index: 0)
-        }
         
         self.encodeDefaultCommands(using: renderCommandEncoder)
         

submodules/TelegramUI/Components/MediaEditor/Sources/VideoTextureSource.swift

@@ -210,11 +210,17 @@ final class VideoTextureSource: NSObject, TextureSource, AVPlayerItemOutputPullD
 
 final class VideoInputPass: DefaultRenderPass {
     private var cachedTexture: MTLTexture?
+    private let scalePass = VideoInputScalePass()
     
     override var fragmentShaderFunctionName: String {
         return "bt709ToRGBFragmentShader"
     }
     
+    override func setup(device: MTLDevice, library: MTLLibrary) {
+        super.setup(device: device, library: library)
+        self.scalePass.setup(device: device, library: library)
+    }
+    
     func processPixelBuffer(_ pixelBuffer: CVPixelBuffer, rotation: TextureRotation, textureCache: CVMetalTextureCache, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
         func textureFromPixelBuffer(_ pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, width: Int, height: Int, plane: Int) -> MTLTexture? {
             var textureRef : CVMetalTexture?
@@ -247,9 +253,6 @@ final class VideoInputPass: DefaultRenderPass {
         }
         
         let (outputWidth, outputHeight) = textureDimensionsForRotation(width: width, height: height, rotation: rotation)
-//        let outputSize = CGSize(width: outputWidth, height: outputHeight).fitted(CGSize(width: 1920.0, height: 1920.0))
-//        outputWidth = Int(outputSize.width)
-//        outputHeight = Int(outputSize.height)
         if self.cachedTexture == nil {
             let textureDescriptor = MTLTextureDescriptor()
             textureDescriptor.textureType = .type2D
@@ -285,6 +288,63 @@ final class VideoInputPass: DefaultRenderPass {
         
         renderCommandEncoder.endEncoding()
         
-        return self.cachedTexture
+        var outputTexture = self.cachedTexture
+        if let texture = outputTexture {
+            outputTexture = self.scalePass.process(input: texture, device: device, commandBuffer: commandBuffer)
+        }
+        return outputTexture
+    }
+}
+
+final class VideoInputScalePass: DefaultRenderPass {
+    private var cachedTexture: MTLTexture?
+    
+    override func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
+        guard max(input.width, input.height) > 1920 else {
+            return input
+        }
+        self.setupVerticesBuffer(device: device)
+                
+        let scaledSize = CGSize(width: input.width, height: input.height).fitted(CGSize(width: 1920.0, height: 1920.0))
+        let width = Int(scaledSize.width)
+        let height = Int(scaledSize.height)
+        
+        if self.cachedTexture == nil || self.cachedTexture?.width != width || self.cachedTexture?.height != height {
+            let textureDescriptor = MTLTextureDescriptor()
+            textureDescriptor.textureType = .type2D
+            textureDescriptor.width = width
+            textureDescriptor.height = height
+            textureDescriptor.pixelFormat = input.pixelFormat
+            textureDescriptor.storageMode = .private
+            textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
+            guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
+                return input
+            }
+            self.cachedTexture = texture
+            texture.label = "scaledVideoTexture"
+        }
+        
+        let renderPassDescriptor = MTLRenderPassDescriptor()
+        renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
+        renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
+        renderPassDescriptor.colorAttachments[0].storeAction = .store
+        renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 1.0)
+        guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
+            return input
+        }
+        
+        renderCommandEncoder.setViewport(MTLViewport(
+            originX: 0, originY: 0,
+            width: Double(width), height: Double(height),
+            znear: -1.0, zfar: 1.0)
+        )
+        
+        renderCommandEncoder.setFragmentTexture(input, index: 0)
+
+        self.encodeDefaultCommands(using: renderCommandEncoder)
+        
+        renderCommandEncoder.endEncoding()
+        
+        return self.cachedTexture!
     }
 }