Animated stickers video scrubbing sync

This commit is contained in:
Ilya Laktyushin 2020-07-25 00:52:31 +03:00
parent c67feac3ce
commit 1bc1a6eea6
13 changed files with 284 additions and 54 deletions

View File

@ -52,6 +52,7 @@ public enum AnimatedStickerMode {
public enum AnimatedStickerPlaybackPosition {
case start
case end
case timestamp(Double)
}
public enum AnimatedStickerPlaybackMode {
@ -83,8 +84,9 @@ public final class AnimatedStickerFrame {
public protocol AnimatedStickerFrameSource: class {
var frameRate: Int { get }
var frameCount: Int { get }
var frameIndex: Int { get }
func takeFrame() -> AnimatedStickerFrame?
func takeFrame(draw: Bool) -> AnimatedStickerFrame?
func skipToEnd()
}
@ -109,7 +111,7 @@ public final class AnimatedStickerCachedFrameSource: AnimatedStickerFrameSource
let height: Int
public let frameRate: Int
public let frameCount: Int
private var frameIndex: Int
public var frameIndex: Int
private let initialOffset: Int
private var offset: Int
var decodeBuffer: Data
@ -179,7 +181,7 @@ public final class AnimatedStickerCachedFrameSource: AnimatedStickerFrameSource
assert(self.queue.isCurrent())
}
public func takeFrame() -> AnimatedStickerFrame? {
public func takeFrame(draw: Bool) -> AnimatedStickerFrame? {
var frameData: Data?
var isLastFrame = false
@ -210,27 +212,29 @@ public final class AnimatedStickerCachedFrameSource: AnimatedStickerFrameSource
self.offset += 4
self.scratchBuffer.withUnsafeMutableBytes { (scratchBytes: UnsafeMutablePointer<UInt8>) -> Void in
self.decodeBuffer.withUnsafeMutableBytes { (decodeBytes: UnsafeMutablePointer<UInt8>) -> Void in
self.frameBuffer.withUnsafeMutableBytes { (frameBytes: UnsafeMutablePointer<UInt8>) -> Void in
compression_decode_buffer(decodeBytes, decodeBufferLength, bytes.advanced(by: self.offset), Int(frameLength), UnsafeMutableRawPointer(scratchBytes), COMPRESSION_LZFSE)
var lhs = UnsafeMutableRawPointer(frameBytes).assumingMemoryBound(to: UInt64.self)
var rhs = UnsafeRawPointer(decodeBytes).assumingMemoryBound(to: UInt64.self)
for _ in 0 ..< decodeBufferLength / 8 {
lhs.pointee = lhs.pointee ^ rhs.pointee
lhs = lhs.advanced(by: 1)
rhs = rhs.advanced(by: 1)
if draw {
self.scratchBuffer.withUnsafeMutableBytes { (scratchBytes: UnsafeMutablePointer<UInt8>) -> Void in
self.decodeBuffer.withUnsafeMutableBytes { (decodeBytes: UnsafeMutablePointer<UInt8>) -> Void in
self.frameBuffer.withUnsafeMutableBytes { (frameBytes: UnsafeMutablePointer<UInt8>) -> Void in
compression_decode_buffer(decodeBytes, decodeBufferLength, bytes.advanced(by: self.offset), Int(frameLength), UnsafeMutableRawPointer(scratchBytes), COMPRESSION_LZFSE)
var lhs = UnsafeMutableRawPointer(frameBytes).assumingMemoryBound(to: UInt64.self)
var rhs = UnsafeRawPointer(decodeBytes).assumingMemoryBound(to: UInt64.self)
for _ in 0 ..< decodeBufferLength / 8 {
lhs.pointee = lhs.pointee ^ rhs.pointee
lhs = lhs.advanced(by: 1)
rhs = rhs.advanced(by: 1)
}
var lhsRest = UnsafeMutableRawPointer(frameBytes).assumingMemoryBound(to: UInt8.self).advanced(by: (decodeBufferLength / 8) * 8)
var rhsRest = UnsafeMutableRawPointer(decodeBytes).assumingMemoryBound(to: UInt8.self).advanced(by: (decodeBufferLength / 8) * 8)
for _ in (decodeBufferLength / 8) * 8 ..< decodeBufferLength {
lhsRest.pointee = rhsRest.pointee ^ lhsRest.pointee
lhsRest = lhsRest.advanced(by: 1)
rhsRest = rhsRest.advanced(by: 1)
}
frameData = Data(bytes: frameBytes, count: decodeBufferLength)
}
var lhsRest = UnsafeMutableRawPointer(frameBytes).assumingMemoryBound(to: UInt8.self).advanced(by: (decodeBufferLength / 8) * 8)
var rhsRest = UnsafeMutableRawPointer(decodeBytes).assumingMemoryBound(to: UInt8.self).advanced(by: (decodeBufferLength / 8) * 8)
for _ in (decodeBufferLength / 8) * 8 ..< decodeBufferLength {
lhsRest.pointee = rhsRest.pointee ^ lhsRest.pointee
lhsRest = lhsRest.advanced(by: 1)
rhsRest = rhsRest.advanced(by: 1)
}
frameData = Data(bytes: frameBytes, count: decodeBufferLength)
}
}
}
@ -247,7 +251,7 @@ public final class AnimatedStickerCachedFrameSource: AnimatedStickerFrameSource
}
}
if let frameData = frameData {
if let frameData = frameData, draw {
return AnimatedStickerFrame(data: frameData, type: .yuva, width: self.width, height: self.height, bytesPerRow: self.bytesPerRow, index: frameIndex, isLastFrame: isLastFrame)
} else {
return nil
@ -271,9 +275,13 @@ private final class AnimatedStickerDirectFrameSource: AnimatedStickerFrameSource
private let bytesPerRow: Int
let frameCount: Int
let frameRate: Int
private var currentFrame: Int
fileprivate var currentFrame: Int
private let animation: LottieInstance
var frameIndex: Int {
return self.currentFrame % self.frameCount
}
init?(queue: Queue, data: Data, width: Int, height: Int) {
self.queue = queue
self.data = data
@ -294,15 +302,19 @@ private final class AnimatedStickerDirectFrameSource: AnimatedStickerFrameSource
assert(self.queue.isCurrent())
}
func takeFrame() -> AnimatedStickerFrame? {
func takeFrame(draw: Bool) -> AnimatedStickerFrame? {
let frameIndex = self.currentFrame % self.frameCount
self.currentFrame += 1
var frameData = Data(count: self.bytesPerRow * self.height)
frameData.withUnsafeMutableBytes { (bytes: UnsafeMutablePointer<UInt8>) -> Void in
memset(bytes, 0, self.bytesPerRow * self.height)
self.animation.renderFrame(with: Int32(frameIndex), into: bytes, width: Int32(self.width), height: Int32(self.height), bytesPerRow: Int32(self.bytesPerRow))
if draw {
var frameData = Data(count: self.bytesPerRow * self.height)
frameData.withUnsafeMutableBytes { (bytes: UnsafeMutablePointer<UInt8>) -> Void in
memset(bytes, 0, self.bytesPerRow * self.height)
self.animation.renderFrame(with: Int32(frameIndex), into: bytes, width: Int32(self.width), height: Int32(self.height), bytesPerRow: Int32(self.bytesPerRow))
}
return AnimatedStickerFrame(data: frameData, type: .argb, width: self.width, height: self.height, bytesPerRow: self.bytesPerRow, index: frameIndex, isLastFrame: frameIndex == self.frameCount - 1)
} else {
return nil
}
return AnimatedStickerFrame(data: frameData, type: .argb, width: self.width, height: self.height, bytesPerRow: self.bytesPerRow, index: frameIndex, isLastFrame: frameIndex == self.frameCount - 1)
}
func skipToEnd() {
@ -326,9 +338,9 @@ public final class AnimatedStickerFrameQueue {
assert(self.queue.isCurrent())
}
public func take() -> AnimatedStickerFrame? {
public func take(draw: Bool) -> AnimatedStickerFrame? {
if self.frames.isEmpty {
if let frame = self.source.takeFrame() {
if let frame = self.source.takeFrame(draw: draw) {
self.frames.append(frame)
}
}
@ -342,7 +354,7 @@ public final class AnimatedStickerFrameQueue {
public func generateFramesIfNeeded() {
if self.frames.isEmpty {
if let frame = self.source.takeFrame() {
if let frame = self.source.takeFrame(draw: true) {
self.frames.append(frame)
}
}
@ -583,7 +595,7 @@ public final class AnimatedStickerNode: ASDisplayNode {
let timer = SwiftSignalKit.Timer(timeout: 1.0 / Double(frameRate), repeat: !firstFrame, completion: {
let maybeFrame = frameQueue.syncWith { frameQueue in
return frameQueue.take()
return frameQueue.take(draw: true)
}
if let maybeFrame = maybeFrame, let frame = maybeFrame {
Queue.mainQueue().async {
@ -654,7 +666,7 @@ public final class AnimatedStickerNode: ASDisplayNode {
let timer = SwiftSignalKit.Timer(timeout: 1.0 / Double(frameRate), repeat: !firstFrame, completion: {
let maybeFrame = frameQueue.syncWith { frameQueue in
return frameQueue.take()
return frameQueue.take(draw: true)
}
if let maybeFrame = maybeFrame, let frame = maybeFrame {
Queue.mainQueue().async {
@ -710,19 +722,25 @@ public final class AnimatedStickerNode: ASDisplayNode {
let directData = self.directData
let cachedData = self.cachedData
let queue = self.queue
let frameSourceHolder = self.frameSource
let timerHolder = self.timer
self.queue.async { [weak self] in
var maybeFrameSource: AnimatedStickerFrameSource?
if let directData = directData {
maybeFrameSource = AnimatedStickerDirectFrameSource(queue: queue, data: directData.0, width: directData.2, height: directData.3)
if position == .end {
maybeFrameSource?.skipToEnd()
}
} else if let (cachedData, cachedDataComplete) = cachedData {
if #available(iOS 9.0, *) {
maybeFrameSource = AnimatedStickerCachedFrameSource(queue: queue, data: cachedData, complete: cachedDataComplete, notifyUpdated: {})
var maybeFrameSource: AnimatedStickerFrameSource? = frameSourceHolder.with { $0 }?.syncWith { $0 }?.value
if case .timestamp = position {
} else {
var maybeFrameSource: AnimatedStickerFrameSource?
if let directData = directData {
maybeFrameSource = AnimatedStickerDirectFrameSource(queue: queue, data: directData.0, width: directData.2, height: directData.3)
if case .end = position {
maybeFrameSource?.skipToEnd()
}
} else if let (cachedData, cachedDataComplete) = cachedData {
if #available(iOS 9.0, *) {
maybeFrameSource = AnimatedStickerCachedFrameSource(queue: queue, data: cachedData, complete: cachedDataComplete, notifyUpdated: {})
}
}
}
guard let frameSource = maybeFrameSource else {
return
}
@ -732,9 +750,31 @@ public final class AnimatedStickerNode: ASDisplayNode {
timerHolder.swap(nil)?.invalidate()
let duration: Double = frameSource.frameRate > 0 ? Double(frameSource.frameCount) / Double(frameSource.frameRate) : 0
let maybeFrame = frameQueue.syncWith { frameQueue in
return frameQueue.take()
var maybeFrame: AnimatedStickerFrame??
if case let .timestamp(timestamp) = position {
var stickerTimestamp = timestamp
while stickerTimestamp > duration {
stickerTimestamp -= duration
}
let targetFrame = Int(stickerTimestamp / duration * Double(frameSource.frameCount))
if targetFrame == frameSource.frameIndex {
return
}
var delta = targetFrame - frameSource.frameIndex
if delta < 0 {
delta = frameSource.frameCount + delta
}
for i in 0 ..< delta {
maybeFrame = frameQueue.syncWith { frameQueue in
return frameQueue.take(draw: i == delta - 1)
}
}
} else {
maybeFrame = frameQueue.syncWith { frameQueue in
return frameQueue.take(draw: true)
}
}
if let maybeFrame = maybeFrame, let frame = maybeFrame {
Queue.mainQueue().async {

View File

@ -85,4 +85,7 @@ typedef enum {
+ (TGPhotoEditorTab)defaultTabsForAvatarIntent;
- (NSTimeInterval)currentTime;
- (void)setMinimalVideoDuration:(NSTimeInterval)duration;
@end

View File

@ -13,7 +13,13 @@
@protocol TGPhotoPaintStickerRenderView <NSObject>
@property (nonatomic, copy) void(^started)(double);
- (void)setIsVisible:(bool)isVisible;
- (void)seekTo:(double)timestamp;
- (void)play;
- (void)pause;
- (void)resetToStart;
- (int64_t)documentId;
- (UIImage *)image;

View File

@ -19,6 +19,7 @@
- (void)setDotVideoView:(UIView *)dotVideoView;
- (void)setDotImage:(UIImage *)dotImage;
@property (nonatomic, assign) NSTimeInterval minimumLength;
@property (nonatomic, assign) NSTimeInterval maximumLength;
@property (nonatomic, assign) bool disableZoom;

View File

@ -95,6 +95,7 @@ typedef enum
if (self != nil)
{
_allowsTrimming = true;
_minimumLength = TGVideoScrubberMinimumTrimDuration;
_currentTimeLabel = [[UILabel alloc] initWithFrame:CGRectMake(8, 4, 100, 15)];
_currentTimeLabel.font = TGSystemFontOfSize(12.0f);
@ -237,7 +238,7 @@ typedef enum
NSTimeInterval duration = trimEndPosition - trimStartPosition;
if (trimEndPosition - trimStartPosition < TGVideoScrubberMinimumTrimDuration)
if (trimEndPosition - trimStartPosition < self.minimumLength)
return;
if (strongSelf.maximumLength > DBL_EPSILON && duration > strongSelf.maximumLength)
@ -300,7 +301,7 @@ typedef enum
NSTimeInterval duration = trimEndPosition - trimStartPosition;
if (trimEndPosition - trimStartPosition < TGVideoScrubberMinimumTrimDuration)
if (trimEndPosition - trimStartPosition < self.minimumLength)
return;
if (strongSelf.maximumLength > DBL_EPSILON && duration > strongSelf.maximumLength)

View File

@ -82,6 +82,7 @@
UIImage *_thumbnailImage;
CMTime _chaseTime;
bool _chaseStart;
bool _chasingTime;
bool _isPlaying;
AVPlayerItem *_playerItem;
@ -367,6 +368,7 @@
if ([self presentedForAvatarCreation] && _item.isVideo) {
_scrubberView = [[TGMediaPickerGalleryVideoScrubber alloc] initWithFrame:CGRectMake(0.0f, 0.0, _portraitToolbarView.frame.size.width, 68.0f)];
_scrubberView.minimumLength = 3.0;
_scrubberView.layer.allowsGroupOpacity = true;
_scrubberView.hasDotPicker = true;
_scrubberView.dataSource = self;
@ -670,6 +672,9 @@
if (strongSelf != nil && !strongSelf->_dismissed) {
[strongSelf->_player seekToTime:startTime];
[strongSelf->_scrubberView setValue:strongSelf.trimStartValue resetPosition:true];
[strongSelf->_fullEntitiesView seekTo:0.0];
[strongSelf->_fullEntitiesView play];
}
}];
}
@ -699,6 +704,11 @@
[_player addObserver:self forKeyPath:@"rate" options:NSKeyValueObservingOptionNew context:nil];
_registeredKeypathObserver = true;
}
[_fullEntitiesView seekTo:0.0];
[_fullEntitiesView play];
} else {
[_fullEntitiesView play];
}
_isPlaying = true;
@ -722,6 +732,8 @@
}
[_scrubberView setIsPlaying:false];
} else {
[_fullEntitiesView pause];
}
_isPlaying = false;
@ -748,6 +760,14 @@
}
}
- (NSTimeInterval)currentTime {
return CMTimeGetSeconds(_player.currentItem.currentTime) - [self trimStartValue];
}
- (void)setMinimalVideoDuration:(NSTimeInterval)duration {
_scrubberView.minimumLength = duration;
}
- (void)seekVideo:(NSTimeInterval)position {
CMTime targetTime = CMTimeMakeWithSeconds(position, NSEC_PER_SEC);
@ -766,6 +786,11 @@
CMTime currentChasingTime = _chaseTime;
[_player.currentItem seekToTime:currentChasingTime toleranceBefore:kCMTimeZero toleranceAfter:kCMTimeZero completionHandler:^(BOOL finished) {
if (!_chaseStart) {
TGDispatchOnMainThread(^{
[_fullEntitiesView seekTo:CMTimeGetSeconds(currentChasingTime) - _scrubberView.trimStartValue];
});
}
if (CMTIME_COMPARE_INLINE(currentChasingTime, ==, _chaseTime)) {
_chasingTime = false;
_chaseTime = kCMTimeInvalid;
@ -2743,7 +2768,7 @@
});
}
- (void)videoScrubber:(TGMediaPickerGalleryVideoScrubber *)__unused videoScrubber valueDidChange:(NSTimeInterval)position
- (void)videoScrubber:(TGMediaPickerGalleryVideoScrubber *)videoScrubber valueDidChange:(NSTimeInterval)position
{
[self seekVideo:position];
}
@ -2780,6 +2805,8 @@
[self startVideoPlayback:true];
[self setPlayButtonHidden:true animated:false];
_chaseStart = false;
}
- (void)videoScrubber:(TGMediaPickerGalleryVideoScrubber *)videoScrubber editingStartValueDidChange:(NSTimeInterval)startValue
@ -2788,6 +2815,12 @@
_resetDotPosition = true;
[self resetDotImage];
}
if (!_chaseStart) {
_chaseStart = true;
[_fullEntitiesView resetToStart];
}
[self seekVideo:startValue];
}

View File

@ -14,6 +14,10 @@
@property (nonatomic, copy) void (^entityRemoved)(TGPhotoPaintEntityView *);
- (void)updateVisibility:(bool)visible;
- (void)seekTo:(double)timestamp;
- (void)play;
- (void)pause;
- (void)resetToStart;
- (UIColor *)colorAtPoint:(CGPoint)point;

View File

@ -40,6 +40,55 @@
}
}
- (void)seekTo:(double)timestamp {
for (TGPhotoPaintEntityView *view in self.subviews)
{
if (![view isKindOfClass:[TGPhotoPaintEntityView class]])
continue;
if ([view isKindOfClass:[TGPhotoStickerEntityView class]]) {
[(TGPhotoStickerEntityView *)view seekTo:timestamp];
}
}
}
- (void)play {
for (TGPhotoPaintEntityView *view in self.subviews)
{
if (![view isKindOfClass:[TGPhotoPaintEntityView class]])
continue;
if ([view isKindOfClass:[TGPhotoStickerEntityView class]]) {
[(TGPhotoStickerEntityView *)view play];
}
}
}
- (void)pause {
for (TGPhotoPaintEntityView *view in self.subviews)
{
if (![view isKindOfClass:[TGPhotoPaintEntityView class]])
continue;
if ([view isKindOfClass:[TGPhotoStickerEntityView class]]) {
[(TGPhotoStickerEntityView *)view pause];
}
}
}
- (void)resetToStart {
for (TGPhotoPaintEntityView *view in self.subviews)
{
if (![view isKindOfClass:[TGPhotoPaintEntityView class]])
continue;
if ([view isKindOfClass:[TGPhotoStickerEntityView class]]) {
[(TGPhotoStickerEntityView *)view resetToStart];
}
}
}
- (BOOL)gestureRecognizer:(UIGestureRecognizer *)__unused gestureRecognizer shouldRecognizeSimultaneouslyWithGestureRecognizer:(UIGestureRecognizer *)__unused otherGestureRecognizer
{
return false;

View File

@ -1196,9 +1196,39 @@ const CGFloat TGPhotoPaintStickerKeyboardSize = 260.0f;
TGPhotoPaintStickerEntity *entity = [[TGPhotoPaintStickerEntity alloc] initWithDocument:document baseSize:[self _stickerBaseSizeForCurrentPainting] animated:animated];
[self _setStickerEntityPosition:entity];
bool hasStickers = false;
for (TGPhotoPaintEntityView *view in _entitiesContainerView.subviews) {
if ([view isKindOfClass:[TGPhotoStickerEntityView class]]) {
hasStickers = true;
break;
}
}
TGPhotoStickerEntityView *stickerView = (TGPhotoStickerEntityView *)[_entitiesContainerView createEntityViewWithEntity:entity];
[self _commonEntityViewSetup:stickerView];
__weak TGPhotoPaintController *weakSelf = self;
__weak TGPhotoStickerEntityView *weakStickerView = stickerView;
stickerView.started = ^(double duration) {
__strong TGPhotoPaintController *strongSelf = weakSelf;
if (strongSelf != nil) {
TGPhotoEditorController *editorController = (TGPhotoEditorController *)self.parentViewController;
if (![editorController isKindOfClass:[TGPhotoEditorController class]])
return;
if (hasStickers) {
[editorController setMinimalVideoDuration:duration];
}
NSTimeInterval currentTime = editorController.currentTime;
__strong TGPhotoStickerEntityView *strongStickerView = weakStickerView;
if (strongStickerView != nil) {
[strongStickerView seekTo:currentTime];
[strongStickerView play];
}
}
};
[self selectEntityView:stickerView];
_entitySelectionView.alpha = 0.0f;

View File

@ -9,6 +9,8 @@
@interface TGPhotoStickerEntityView : TGPhotoPaintEntityView
@property (nonatomic, copy) void(^started)(double);
@property (nonatomic, readonly) TGPhotoPaintStickerEntity *entity;
@property (nonatomic, readonly) bool isMirrored;
@ -17,6 +19,10 @@
- (UIImage *)image;
- (void)updateVisibility:(bool)visible;
- (void)seekTo:(double)timestamp;
- (void)play;
- (void)pause;
- (void)resetToStart;
- (CGRect)realBounds;

View File

@ -55,6 +55,13 @@ const CGFloat TGPhotoStickerSelectionViewHandleSide = 30.0f;
_mirrored = entity.isMirrored;
_stickerView = [context stickerViewForDocument:entity.document];
__weak TGPhotoStickerEntityView *weakSelf = self;
_stickerView.started = ^(double duration) {
__strong TGPhotoStickerEntityView *strongSelf = weakSelf;
if (strongSelf != nil && strongSelf.started != nil)
strongSelf.started(duration);
};
[self addSubview:_stickerView];
_document = entity.document;
@ -178,6 +185,22 @@ const CGFloat TGPhotoStickerSelectionViewHandleSide = 30.0f;
[_stickerView setIsVisible:visible];
}
- (void)seekTo:(double)timestamp {
[_stickerView seekTo:timestamp];
}
- (void)play {
[_stickerView play];
}
- (void)pause {
[_stickerView pause];
}
- (void)resetToStart {
[_stickerView resetToStart];
}
@end

View File

@ -10,6 +10,8 @@ import StickerResources
import LegacyComponents
class LegacyPaintStickerView: UIView, TGPhotoPaintStickerRenderView {
var started: ((Double) -> Void)?
private let context: AccountContext
private let file: TelegramMediaFile
private var currentSize: CGSize?
@ -63,8 +65,16 @@ class LegacyPaintStickerView: UIView, TGPhotoPaintStickerRenderView {
if self.animationNode == nil {
let animationNode = AnimatedStickerNode()
self.animationNode = animationNode
animationNode.started = { [weak self] in
animationNode.started = { [weak self, weak animationNode] in
self?.imageNode.isHidden = true
if let animationNode = animationNode {
let _ = (animationNode.status
|> take(1)
|> deliverOnMainQueue).start(next: { [weak self] status in
self?.started?(status.duration)
})
}
}
self.addSubnode(animationNode)
}
@ -115,6 +125,30 @@ class LegacyPaintStickerView: UIView, TGPhotoPaintStickerRenderView {
}
}
func seek(to timestamp: Double) {
self.isVisible = false
self.isPlaying = false
self.animationNode?.seekTo(.timestamp(timestamp))
}
func play() {
self.isVisible = true
self.isPlaying = true
self.animationNode?.play()
}
func pause() {
self.isVisible = false
self.isPlaying = false
self.animationNode?.pause()
}
func resetToStart() {
self.isVisible = false
self.isPlaying = false
self.animationNode?.seekTo(.timestamp(0.0))
}
override func layoutSubviews() {
super.layoutSubviews()

View File

@ -188,8 +188,8 @@ private class LegacyPaintStickerEntity: LegacyPaintEntity {
let maybeFrame = frameQueue.syncWith { frameQueue -> AnimatedStickerFrame? in
var frame: AnimatedStickerFrame?
for _ in 0 ..< delta {
frame = frameQueue.take()
for i in 0 ..< delta {
frame = frameQueue.take(draw: i == delta - 1)
}
return frame
}