Camera improvements

submodules/Camera/Sources/CameraPreviewView.swift

@@ -6,6 +6,7 @@ import SwiftSignalKit
 import Metal
 import MetalKit
 import CoreMedia
+import Vision
 
 public class CameraPreviewView: MTKView {
     private let queue = DispatchQueue(label: "CameraPreview", qos: .userInitiated, attributes: [], autoreleaseFrequency: .workItem)
@@ -302,4 +303,198 @@ public class CameraPreviewView: MTKView {
         commandBuffer.present(drawable)
         commandBuffer.commit()
     }
+    
+    
+    var captureDeviceResolution: CGSize = CGSize() {
+        didSet {
+            if oldValue.width.isZero, !self.captureDeviceResolution.width.isZero {
+                Queue.mainQueue().async {
+                    self.setupVisionDrawingLayers()
+                }
+            }
+        }
+    }
+    var detectionOverlayLayer: CALayer?
+    var detectedFaceRectangleShapeLayer: CAShapeLayer?
+    var detectedFaceLandmarksShapeLayer: CAShapeLayer?
+    
+    func drawFaceObservations(_ faceObservations: [VNFaceObservation]) {
+        guard let faceRectangleShapeLayer = self.detectedFaceRectangleShapeLayer,
+            let faceLandmarksShapeLayer = self.detectedFaceLandmarksShapeLayer
+            else {
+            return
+        }
+        
+        CATransaction.begin()
+        
+        CATransaction.setValue(NSNumber(value: true), forKey: kCATransactionDisableActions)
+        
+        self.detectionOverlayLayer?.isHidden = faceObservations.isEmpty
+        
+        let faceRectanglePath = CGMutablePath()
+        let faceLandmarksPath = CGMutablePath()
+        
+        for faceObservation in faceObservations {
+            self.addIndicators(to: faceRectanglePath,
+                               faceLandmarksPath: faceLandmarksPath,
+                               for: faceObservation)
+        }
+        
+        faceRectangleShapeLayer.path = faceRectanglePath
+        faceLandmarksShapeLayer.path = faceLandmarksPath
+        
+        self.updateLayerGeometry()
+        
+        CATransaction.commit()
+    }
+    
+    fileprivate func addPoints(in landmarkRegion: VNFaceLandmarkRegion2D, to path: CGMutablePath, applying affineTransform: CGAffineTransform, closingWhenComplete closePath: Bool) {
+        let pointCount = landmarkRegion.pointCount
+        if pointCount > 1 {
+            let points: [CGPoint] = landmarkRegion.normalizedPoints
+            path.move(to: points[0], transform: affineTransform)
+            path.addLines(between: points, transform: affineTransform)
+            if closePath {
+                path.addLine(to: points[0], transform: affineTransform)
+                path.closeSubpath()
+            }
+        }
+    }
+    
+    fileprivate func addIndicators(to faceRectanglePath: CGMutablePath, faceLandmarksPath: CGMutablePath, for faceObservation: VNFaceObservation) {
+        let displaySize = self.captureDeviceResolution
+        
+        let faceBounds = VNImageRectForNormalizedRect(faceObservation.boundingBox, Int(displaySize.width), Int(displaySize.height))
+        faceRectanglePath.addRect(faceBounds)
+        
+        if let landmarks = faceObservation.landmarks {
+            let affineTransform = CGAffineTransform(translationX: faceBounds.origin.x, y: faceBounds.origin.y)
+                .scaledBy(x: faceBounds.size.width, y: faceBounds.size.height)
+            
+            let openLandmarkRegions: [VNFaceLandmarkRegion2D?] = [
+                landmarks.leftEyebrow,
+                landmarks.rightEyebrow,
+                landmarks.faceContour,
+                landmarks.noseCrest,
+                landmarks.medianLine
+            ]
+            for openLandmarkRegion in openLandmarkRegions where openLandmarkRegion != nil {
+                self.addPoints(in: openLandmarkRegion!, to: faceLandmarksPath, applying: affineTransform, closingWhenComplete: false)
+            }
+            
+            let closedLandmarkRegions: [VNFaceLandmarkRegion2D?] = [
+                landmarks.leftEye,
+                landmarks.rightEye,
+                landmarks.outerLips,
+                landmarks.innerLips,
+                landmarks.nose
+            ]
+            for closedLandmarkRegion in closedLandmarkRegions where closedLandmarkRegion != nil {
+                self.addPoints(in: closedLandmarkRegion!, to: faceLandmarksPath, applying: affineTransform, closingWhenComplete: true)
+            }
+        }
+    }
+    
+    fileprivate func radiansForDegrees(_ degrees: CGFloat) -> CGFloat {
+        return CGFloat(Double(degrees) * Double.pi / 180.0)
+    }
+    
+    fileprivate func updateLayerGeometry() {
+        guard let overlayLayer = self.detectionOverlayLayer else {
+            return
+        }
+        
+        CATransaction.setValue(NSNumber(value: true), forKey: kCATransactionDisableActions)
+        
+        let videoPreviewRect = self.bounds
+        
+        var rotation: CGFloat
+        var scaleX: CGFloat
+        var scaleY: CGFloat
+        
+        // Rotate the layer into screen orientation.
+        switch UIDevice.current.orientation {
+        case .portraitUpsideDown:
+            rotation = 180
+            scaleX = videoPreviewRect.width / captureDeviceResolution.width
+            scaleY = videoPreviewRect.height / captureDeviceResolution.height
+            
+        case .landscapeLeft:
+            rotation = 90
+            scaleX = videoPreviewRect.height / captureDeviceResolution.width
+            scaleY = scaleX
+            
+        case .landscapeRight:
+            rotation = -90
+            scaleX = videoPreviewRect.height / captureDeviceResolution.width
+            scaleY = scaleX
+            
+        default:
+            rotation = 0
+            scaleX = videoPreviewRect.width / captureDeviceResolution.width
+            scaleY = videoPreviewRect.height / captureDeviceResolution.height
+        }
+        
+        // Scale and mirror the image to ensure upright presentation.
+        let affineTransform = CGAffineTransform(rotationAngle: radiansForDegrees(rotation))
+            .scaledBy(x: scaleX, y: -scaleY)
+        overlayLayer.setAffineTransform(affineTransform)
+        
+        // Cover entire screen UI.
+        let rootLayerBounds = self.bounds
+        overlayLayer.position = CGPoint(x: rootLayerBounds.midX, y: rootLayerBounds.midY)
+    }
+    
+    fileprivate func setupVisionDrawingLayers() {
+        let captureDeviceResolution = self.captureDeviceResolution
+        let rootLayer = self.layer
+        
+        let captureDeviceBounds = CGRect(x: 0,
+                                         y: 0,
+                                         width: captureDeviceResolution.width,
+                                         height: captureDeviceResolution.height)
+        
+        let captureDeviceBoundsCenterPoint = CGPoint(x: captureDeviceBounds.midX,
+                                                     y: captureDeviceBounds.midY)
+        
+        let normalizedCenterPoint = CGPoint(x: 0.5, y: 0.5)
+        
+        let overlayLayer = CALayer()
+        overlayLayer.name = "DetectionOverlay"
+        overlayLayer.masksToBounds = true
+        overlayLayer.anchorPoint = normalizedCenterPoint
+        overlayLayer.bounds = captureDeviceBounds
+        overlayLayer.position = CGPoint(x: rootLayer.bounds.midX, y: rootLayer.bounds.midY)
+        
+        let faceRectangleShapeLayer = CAShapeLayer()
+        faceRectangleShapeLayer.name = "RectangleOutlineLayer"
+        faceRectangleShapeLayer.bounds = captureDeviceBounds
+        faceRectangleShapeLayer.anchorPoint = normalizedCenterPoint
+        faceRectangleShapeLayer.position = captureDeviceBoundsCenterPoint
+        faceRectangleShapeLayer.fillColor = nil
+        faceRectangleShapeLayer.strokeColor = UIColor.green.withAlphaComponent(0.2).cgColor
+        faceRectangleShapeLayer.lineWidth = 2
+        
+        let faceLandmarksShapeLayer = CAShapeLayer()
+        faceLandmarksShapeLayer.name = "FaceLandmarksLayer"
+        faceLandmarksShapeLayer.bounds = captureDeviceBounds
+        faceLandmarksShapeLayer.anchorPoint = normalizedCenterPoint
+        faceLandmarksShapeLayer.position = captureDeviceBoundsCenterPoint
+        faceLandmarksShapeLayer.fillColor = nil
+        faceLandmarksShapeLayer.strokeColor = UIColor.white.withAlphaComponent(0.7).cgColor
+        faceLandmarksShapeLayer.lineWidth = 2
+        faceLandmarksShapeLayer.shadowOpacity = 0.7
+        faceLandmarksShapeLayer.shadowRadius = 2
+        
+        overlayLayer.addSublayer(faceRectangleShapeLayer)
+        faceRectangleShapeLayer.addSublayer(faceLandmarksShapeLayer)
+        self.layer.addSublayer(overlayLayer)
+        
+        self.detectionOverlayLayer = overlayLayer
+        self.detectedFaceRectangleShapeLayer = faceRectangleShapeLayer
+        self.detectedFaceLandmarksShapeLayer = faceLandmarksShapeLayer
+        
+        self.updateLayerGeometry()
+    }
+
 }