Swiftgram/submodules/Camera/Sources/CameraOutput.swift
2024-01-13 21:46:02 +04:00

573 lines
24 KiB
Swift

import Foundation
import AVFoundation
import UIKit
import SwiftSignalKit
import CoreImage
import Vision
import VideoToolbox
import TelegramCore
public enum VideoCaptureResult: Equatable {
public struct Result {
public let path: String
public let thumbnail: UIImage
public let isMirrored: Bool
public let dimensions: CGSize
}
case finished(main: Result, additional: Result?, duration: Double, positionChangeTimestamps: [(Bool, Double)], captureTimestamp: Double)
case failed
public static func == (lhs: VideoCaptureResult, rhs: VideoCaptureResult) -> Bool {
switch lhs {
case .failed:
if case .failed = rhs {
return true
} else {
return false
}
case let .finished(_, _, lhsDuration, lhsChangeTimestamps, lhsTimestamp):
if case let .finished(_, _, rhsDuration, rhsChangeTimestamps, rhsTimestamp) = rhs, lhsDuration == rhsDuration, lhsTimestamp == rhsTimestamp {
if lhsChangeTimestamps.count != rhsChangeTimestamps.count {
return false
}
return true
} else {
return false
}
}
}
}
public struct CameraCode: Equatable {
public enum CodeType {
case qr
}
public let type: CodeType
public let message: String
public let corners: [CGPoint]
public init(type: CameraCode.CodeType, message: String, corners: [CGPoint]) {
self.type = type
self.message = message
self.corners = corners
}
public var boundingBox: CGRect {
let x = self.corners.map { $0.x }
let y = self.corners.map { $0.y }
if let minX = x.min(), let minY = y.min(), let maxX = x.max(), let maxY = y.max() {
return CGRect(x: minX, y: minY, width: abs(maxX - minX), height: abs(maxY - minY))
}
return CGRect.null
}
public static func == (lhs: CameraCode, rhs: CameraCode) -> Bool {
if lhs.type != rhs.type {
return false
}
if lhs.message != rhs.message {
return false
}
if lhs.corners != rhs.corners {
return false
}
return true
}
}
final class CameraOutput: NSObject {
let exclusive: Bool
let ciContext: CIContext
let isVideoMessage: Bool
let photoOutput = AVCapturePhotoOutput()
let videoOutput = AVCaptureVideoDataOutput()
let audioOutput = AVCaptureAudioDataOutput()
let metadataOutput = AVCaptureMetadataOutput()
private var photoConnection: AVCaptureConnection?
private var videoConnection: AVCaptureConnection?
private var previewConnection: AVCaptureConnection?
private var roundVideoFilter: CameraRoundVideoFilter?
private let semaphore = DispatchSemaphore(value: 1)
private let queue = DispatchQueue(label: "")
private let metadataQueue = DispatchQueue(label: "")
private var photoCaptureRequests: [Int64: PhotoCaptureContext] = [:]
private var videoRecorder: VideoRecorder?
var processSampleBuffer: ((CMSampleBuffer, CVImageBuffer, AVCaptureConnection) -> Void)?
var processAudioBuffer: ((CMSampleBuffer) -> Void)?
var processCodes: (([CameraCode]) -> Void)?
init(exclusive: Bool, ciContext: CIContext, use32BGRA: Bool = false) {
self.exclusive = exclusive
self.ciContext = ciContext
self.isVideoMessage = use32BGRA
super.init()
if #available(iOS 13.0, *) {
self.photoOutput.maxPhotoQualityPrioritization = .balanced
}
self.videoOutput.alwaysDiscardsLateVideoFrames = false
self.videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey: use32BGRA ? kCVPixelFormatType_32BGRA : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] as [String : Any]
}
deinit {
self.videoOutput.setSampleBufferDelegate(nil, queue: nil)
self.audioOutput.setSampleBufferDelegate(nil, queue: nil)
}
func configure(for session: CameraSession, device: CameraDevice, input: CameraInput, previewView: CameraSimplePreviewView?, audio: Bool, photo: Bool, metadata: Bool) {
if session.session.canAddOutput(self.videoOutput) {
if session.hasMultiCam {
session.session.addOutputWithNoConnections(self.videoOutput)
} else {
session.session.addOutput(self.videoOutput)
}
self.videoOutput.setSampleBufferDelegate(self, queue: self.queue)
} else {
Logger.shared.log("Camera", "Can't add video output")
}
if audio, session.session.canAddOutput(self.audioOutput) {
session.session.addOutput(self.audioOutput)
self.audioOutput.setSampleBufferDelegate(self, queue: self.queue)
}
if photo, session.session.canAddOutput(self.photoOutput) {
if session.hasMultiCam {
session.session.addOutputWithNoConnections(self.photoOutput)
} else {
session.session.addOutput(self.photoOutput)
}
} else {
Logger.shared.log("Camera", "Can't add photo output")
}
if metadata, session.session.canAddOutput(self.metadataOutput) {
session.session.addOutput(self.metadataOutput)
self.metadataOutput.setMetadataObjectsDelegate(self, queue: self.metadataQueue)
if self.metadataOutput.availableMetadataObjectTypes.contains(.qr) {
self.metadataOutput.metadataObjectTypes = [.qr]
}
}
if #available(iOS 13.0, *), session.hasMultiCam {
if let device = device.videoDevice, let ports = input.videoInput?.ports(for: AVMediaType.video, sourceDeviceType: device.deviceType, sourceDevicePosition: device.position) {
if let previewView {
let previewConnection = AVCaptureConnection(inputPort: ports.first!, videoPreviewLayer: previewView.videoPreviewLayer)
if session.session.canAddConnection(previewConnection) {
session.session.addConnection(previewConnection)
self.previewConnection = previewConnection
} else {
Logger.shared.log("Camera", "Can't add preview connection")
}
}
let videoConnection = AVCaptureConnection(inputPorts: ports, output: self.videoOutput)
if session.session.canAddConnection(videoConnection) {
session.session.addConnection(videoConnection)
self.videoConnection = videoConnection
} else {
Logger.shared.log("Camera", "Can't add video connection")
}
if photo {
let photoConnection = AVCaptureConnection(inputPorts: ports, output: self.photoOutput)
if session.session.canAddConnection(photoConnection) {
session.session.addConnection(photoConnection)
self.photoConnection = photoConnection
}
}
} else {
Logger.shared.log("Camera", "Can't get video port")
}
}
}
func invalidate(for session: CameraSession) {
if #available(iOS 13.0, *) {
if let previewConnection = self.previewConnection {
if session.session.connections.contains(where: { $0 === previewConnection }) {
session.session.removeConnection(previewConnection)
}
self.previewConnection = nil
}
if let videoConnection = self.videoConnection {
if session.session.connections.contains(where: { $0 === videoConnection }) {
session.session.removeConnection(videoConnection)
}
self.videoConnection = nil
}
if let photoConnection = self.photoConnection {
if session.session.connections.contains(where: { $0 === photoConnection }) {
session.session.removeConnection(photoConnection)
}
self.photoConnection = nil
}
}
if session.session.outputs.contains(where: { $0 === self.videoOutput }) {
session.session.removeOutput(self.videoOutput)
}
if session.session.outputs.contains(where: { $0 === self.audioOutput }) {
session.session.removeOutput(self.audioOutput)
}
if session.session.outputs.contains(where: { $0 === self.photoOutput }) {
session.session.removeOutput(self.photoOutput)
}
if session.session.outputs.contains(where: { $0 === self.metadataOutput }) {
session.session.removeOutput(self.metadataOutput)
}
}
func configureVideoStabilization() {
if let videoDataOutputConnection = self.videoOutput.connection(with: .video) {
if videoDataOutputConnection.isVideoStabilizationSupported {
videoDataOutputConnection.preferredVideoStabilizationMode = .standard
// videoDataOutputConnection.preferredVideoStabilizationMode = self.isVideoMessage ? .cinematic : .standard
}
}
}
var isFlashActive: Signal<Bool, NoError> {
return Signal { [weak self] subscriber in
guard let self else {
return EmptyDisposable
}
subscriber.putNext(self.photoOutput.isFlashScene)
let observer = self.photoOutput.observe(\.isFlashScene, options: [.new], changeHandler: { device, _ in
subscriber.putNext(self.photoOutput.isFlashScene)
})
return ActionDisposable {
observer.invalidate()
}
}
|> distinctUntilChanged
}
func takePhoto(orientation: AVCaptureVideoOrientation, flashMode: AVCaptureDevice.FlashMode) -> Signal<PhotoCaptureResult, NoError> {
var mirror = false
if let connection = self.photoOutput.connection(with: .video) {
connection.videoOrientation = orientation
if #available(iOS 13.0, *) {
mirror = connection.inputPorts.first?.sourceDevicePosition == .front
}
}
let settings = AVCapturePhotoSettings(format: [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)])
settings.flashMode = mirror ? .off : flashMode
if let previewPhotoPixelFormatType = settings.availablePreviewPhotoPixelFormatTypes.first {
settings.previewPhotoFormat = [kCVPixelBufferPixelFormatTypeKey as String: previewPhotoPixelFormatType]
}
if #available(iOS 13.0, *) {
if self.photoOutput.maxPhotoQualityPrioritization != .speed {
settings.photoQualityPrioritization = .balanced
} else {
settings.photoQualityPrioritization = .speed
}
}
let uniqueId = settings.uniqueID
let photoCapture = PhotoCaptureContext(ciContext: self.ciContext, settings: settings, orientation: orientation, mirror: mirror)
self.photoCaptureRequests[uniqueId] = photoCapture
self.photoOutput.capturePhoto(with: settings, delegate: photoCapture)
return photoCapture.signal
|> afterDisposed { [weak self] in
self?.photoCaptureRequests.removeValue(forKey: uniqueId)
}
}
var isRecording: Bool {
return self.videoRecorder != nil
}
enum RecorderMode {
case `default`
case roundVideo
case dualCamera
}
private var currentMode: RecorderMode = .default
private var recordingCompletionPipe = ValuePipe<VideoCaptureResult>()
func startRecording(mode: RecorderMode, position: Camera.Position? = nil, orientation: AVCaptureVideoOrientation, additionalOutput: CameraOutput? = nil) -> Signal<CameraRecordingData, NoError> {
guard self.videoRecorder == nil else {
return .complete()
}
self.currentMode = mode
self.lastSampleTimestamp = nil
let codecType: AVVideoCodecType
if case .roundVideo = mode {
codecType = .h264
} else {
if hasHEVCHardwareEncoder {
codecType = .hevc
} else {
codecType = .h264
}
}
guard var videoSettings = self.videoOutput.recommendedVideoSettings(forVideoCodecType: codecType, assetWriterOutputFileType: .mp4) else {
return .complete()
}
var dimensions: CGSize = CGSize(width: 1080, height: 1920)
if orientation == .landscapeLeft || orientation == .landscapeRight {
dimensions = CGSize(width: 1920, height: 1080)
}
var orientation = orientation
if case .roundVideo = mode {
videoSettings[AVVideoWidthKey] = 400
videoSettings[AVVideoHeightKey] = 400
dimensions = CGSize(width: 400, height: 400)
orientation = .landscapeRight
}
let audioSettings = self.audioOutput.recommendedAudioSettingsForAssetWriter(writingTo: .mp4) ?? [:]
let outputFileName = NSUUID().uuidString
let outputFilePath = NSTemporaryDirectory() + outputFileName + ".mp4"
let outputFileURL = URL(fileURLWithPath: outputFilePath)
let videoRecorder = VideoRecorder(
configuration: VideoRecorder.Configuration(videoSettings: videoSettings, audioSettings: audioSettings),
ciContext: self.ciContext,
orientation: orientation,
fileUrl: outputFileURL,
completion: { [weak self] result in
guard let self else {
return
}
if case let .success(transitionImage, duration, positionChangeTimestamps) = result {
self.recordingCompletionPipe.putNext(
.finished(
main: VideoCaptureResult.Result(
path: outputFilePath,
thumbnail: transitionImage ?? UIImage(),
isMirrored: false,
dimensions: dimensions
),
additional: nil,
duration: duration,
positionChangeTimestamps: positionChangeTimestamps.map { ($0 == .front, $1) },
captureTimestamp: CACurrentMediaTime()
)
)
} else {
self.recordingCompletionPipe.putNext(.failed)
}
}
)
videoRecorder?.start()
self.videoRecorder = videoRecorder
if case .dualCamera = mode, let position {
videoRecorder?.markPositionChange(position: position, time: .zero)
} else if case .roundVideo = mode {
additionalOutput?.masterOutput = self
}
return Signal { subscriber in
let timer = SwiftSignalKit.Timer(timeout: 0.033, repeat: true, completion: { [weak videoRecorder] in
let recordingData = CameraRecordingData(duration: videoRecorder?.duration ?? 0.0, filePath: outputFilePath)
subscriber.putNext(recordingData)
}, queue: Queue.mainQueue())
timer.start()
return ActionDisposable {
timer.invalidate()
}
}
}
func stopRecording() -> Signal<VideoCaptureResult, NoError> {
guard let videoRecorder = self.videoRecorder, videoRecorder.isRecording else {
return .complete()
}
videoRecorder.stop()
return self.recordingCompletionPipe.signal()
|> take(1)
|> afterDisposed {
self.videoRecorder = nil
}
}
var transitionImage: UIImage? {
return self.videoRecorder?.transitionImage
}
private weak var masterOutput: CameraOutput?
private var lastSampleTimestamp: CMTime?
func processVideoRecording(_ sampleBuffer: CMSampleBuffer, fromAdditionalOutput: Bool) {
guard let formatDescriptor = CMSampleBufferGetFormatDescription(sampleBuffer) else {
return
}
let type = CMFormatDescriptionGetMediaType(formatDescriptor)
if let videoRecorder = self.videoRecorder, videoRecorder.isRecording {
if case .roundVideo = self.currentMode, type == kCMMediaType_Video {
var transitionFactor: CGFloat = 0.0
let currentTimestamp = CACurrentMediaTime()
let duration: Double = 0.2
if case .front = self.currentPosition {
transitionFactor = 1.0
if self.lastSwitchTimestamp > 0.0, currentTimestamp - self.lastSwitchTimestamp < duration {
transitionFactor = max(0.0, (currentTimestamp - self.lastSwitchTimestamp) / duration)
}
} else {
transitionFactor = 0.0
if self.lastSwitchTimestamp > 0.0, currentTimestamp - self.lastSwitchTimestamp < duration {
transitionFactor = 1.0 - max(0.0, (currentTimestamp - self.lastSwitchTimestamp) / duration)
}
}
if let processedSampleBuffer = self.processRoundVideoSampleBuffer(sampleBuffer, additional: fromAdditionalOutput, transitionFactor: transitionFactor) {
let presentationTime = CMSampleBufferGetPresentationTimeStamp(processedSampleBuffer)
if let lastSampleTimestamp = self.lastSampleTimestamp, lastSampleTimestamp > presentationTime {
} else {
if (transitionFactor == 1.0 && fromAdditionalOutput) || (transitionFactor == 0.0 && !fromAdditionalOutput) || (transitionFactor > 0.0 && transitionFactor < 1.0) {
videoRecorder.appendSampleBuffer(processedSampleBuffer)
self.lastSampleTimestamp = presentationTime
}
}
} else {
videoRecorder.appendSampleBuffer(sampleBuffer)
}
} else {
videoRecorder.appendSampleBuffer(sampleBuffer)
}
}
}
private func processRoundVideoSampleBuffer(_ sampleBuffer: CMSampleBuffer, additional: Bool, transitionFactor: CGFloat) -> CMSampleBuffer? {
guard let videoPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer), let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) else {
return nil
}
self.semaphore.wait()
let mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription)
let extensions = CMFormatDescriptionGetExtensions(formatDescription) as! [String: Any]
var updatedExtensions = extensions
updatedExtensions["CVBytesPerRow"] = 400 * 4
var newFormatDescription: CMFormatDescription?
var status = CMVideoFormatDescriptionCreate(allocator: nil, codecType: mediaSubType, width: 400, height: 400, extensions: updatedExtensions as CFDictionary, formatDescriptionOut: &newFormatDescription)
guard status == noErr, let newFormatDescription else {
return nil
}
let filter: CameraRoundVideoFilter
if let current = self.roundVideoFilter {
filter = current
} else {
filter = CameraRoundVideoFilter(ciContext: self.ciContext)
self.roundVideoFilter = filter
}
if !filter.isPrepared {
filter.prepare(with: newFormatDescription, outputRetainedBufferCountHint: 3)
}
guard let newPixelBuffer = filter.render(pixelBuffer: videoPixelBuffer, additional: additional, transitionFactor: transitionFactor) else {
self.semaphore.signal()
return nil
}
var sampleTimingInfo: CMSampleTimingInfo = .invalid
CMSampleBufferGetSampleTimingInfo(sampleBuffer, at: 0, timingInfoOut: &sampleTimingInfo)
var newSampleBuffer: CMSampleBuffer?
status = CMSampleBufferCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: newPixelBuffer,
dataReady: true,
makeDataReadyCallback: nil,
refcon: nil,
formatDescription: newFormatDescription,
sampleTiming: &sampleTimingInfo,
sampleBufferOut: &newSampleBuffer
)
if status == noErr, let newSampleBuffer {
self.semaphore.signal()
return newSampleBuffer
}
self.semaphore.signal()
return nil
}
private var currentPosition: Camera.Position = .front
private var lastSwitchTimestamp: Double = 0.0
func markPositionChange(position: Camera.Position) {
self.currentPosition = position
self.lastSwitchTimestamp = CACurrentMediaTime()
if let videoRecorder = self.videoRecorder {
videoRecorder.markPositionChange(position: position)
}
}
}
extension CameraOutput: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard CMSampleBufferDataIsReady(sampleBuffer) else {
return
}
if let videoPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
self.processSampleBuffer?(sampleBuffer, videoPixelBuffer, connection)
} else {
// self.processAudioBuffer?(sampleBuffer)
}
if let masterOutput = self.masterOutput {
masterOutput.processVideoRecording(sampleBuffer, fromAdditionalOutput: true)
} else {
self.processVideoRecording(sampleBuffer, fromAdditionalOutput: false)
}
}
func captureOutput(_ output: AVCaptureOutput, didDrop sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
}
}
extension CameraOutput: AVCaptureMetadataOutputObjectsDelegate {
func metadataOutput(_ output: AVCaptureMetadataOutput, didOutput metadataObjects: [AVMetadataObject], from connection: AVCaptureConnection) {
let codes: [CameraCode] = metadataObjects.filter { $0.type == .qr }.compactMap { object in
if let object = object as? AVMetadataMachineReadableCodeObject, let stringValue = object.stringValue, !stringValue.isEmpty {
#if targetEnvironment(simulator)
return CameraCode(type: .qr, message: stringValue, corners: [CGPoint(), CGPoint(), CGPoint(), CGPoint()])
#else
return CameraCode(type: .qr, message: stringValue, corners: object.corners)
#endif
} else {
return nil
}
}
self.processCodes?(codes)
}
}
private let hasHEVCHardwareEncoder: Bool = {
let spec: [CFString: Any] = [:]
var outID: CFString?
var properties: CFDictionary?
let result = VTCopySupportedPropertyDictionaryForEncoder(width: 1920, height: 1080, codecType: kCMVideoCodecType_HEVC, encoderSpecification: spec as CFDictionary, encoderIDOut: &outID, supportedPropertiesOut: &properties)
if result == kVTCouldNotFindVideoEncoderErr {
return false
}
return result == noErr
}()