diff --git a/NextcloudTalk/Calls/CallPiPViewController.swift b/NextcloudTalk/Calls/CallPiPViewController.swift new file mode 100644 index 000000000..96ff43b89 --- /dev/null +++ b/NextcloudTalk/Calls/CallPiPViewController.swift @@ -0,0 +1,457 @@ +// +// SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors +// SPDX-License-Identifier: GPL-3.0-or-later +// + +import AVKit +import Foundation +import UIKit +import WebRTC + +// Renders remote RTCVideoFrames into an AVSampleBufferDisplayLayer. +// Metal (RTCMTLVideoView) is not allowed to render while the app is in the background, +// AVSampleBufferDisplayLayer is, which makes it the only way to show remote video +// in the Picture in Picture window. +class SampleBufferVideoRenderView: UIView, RTCVideoRenderer { + + private class DisplayLayerView: UIView { + override class var layerClass: AnyClass { + return AVSampleBufferDisplayLayer.self + } + + var sampleBufferLayer: AVSampleBufferDisplayLayer { + // swiftlint:disable:next force_cast + return layer as! AVSampleBufferDisplayLayer + } + } + + // Called on the main queue whenever the (rotated) size of the rendered video changes + public var onVideoSizeChanged: ((CGSize) -> Void)? + + // Mirrors the rendered video, as it is expected for the own video of the front camera + public var isMirrored = false { + didSet { + guard isMirrored != oldValue else { return } + + DispatchQueue.main.async { + self.transform = self.isMirrored ? CGAffineTransform(scaleX: -1, y: 1) : .identity + } + } + } + + private let displayView = DisplayLayerView() + private var videoRotation = RTCVideoRotation._0 + private var videoSize: CGSize = .zero + + override init(frame: CGRect) { + super.init(frame: frame) + + displayView.sampleBufferLayer.videoGravity = .resizeAspect + self.addSubview(displayView) + } + + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override func layoutSubviews() { + super.layoutSubviews() + self.layoutDisplayView() + } + + // The frames of the sample buffer layer are not rotated, so the rotation + // of the video frames is applied as a transform on the layer's view + private func layoutDisplayView() { + let isRotatedSideways = videoRotation == ._90 || videoRotation == ._270 + + displayView.center = CGPoint(x: bounds.midX, y: bounds.midY) + displayView.bounds = isRotatedSideways ? CGRect(x: 0, y: 0, width: bounds.height, height: bounds.width) : bounds + + switch videoRotation { + case ._90: + displayView.transform = CGAffineTransform(rotationAngle: .pi / 2) + case ._180: + displayView.transform = CGAffineTransform(rotationAngle: .pi) + case ._270: + displayView.transform = CGAffineTransform(rotationAngle: -.pi / 2) + default: + displayView.transform = .identity + } + } + + public func flush() { + DispatchQueue.main.async { + self.displayView.sampleBufferLayer.flushAndRemoveImage() + } + } + + // MARK: - RTCVideoRenderer + + public func setSize(_ size: CGSize) { + // The size reported here does not include the frame rotation, + // so the rotated size is determined in renderFrame instead + } + + public func renderFrame(_ frame: RTCVideoFrame?) { + // Called on the WebRTC decoding thread + guard let frame, + let pixelBuffer = Self.pixelBuffer(from: frame.buffer), + let sampleBuffer = Self.sampleBuffer(from: pixelBuffer) + else { return } + + let rotation = frame.rotation + var size = CGSize(width: CGFloat(frame.width), height: CGFloat(frame.height)) + + if rotation == ._90 || rotation == ._270 { + size = CGSize(width: size.height, height: size.width) + } + + DispatchQueue.main.async { + if self.displayView.sampleBufferLayer.status == .failed { + self.displayView.sampleBufferLayer.flush() + } + + if rotation != self.videoRotation { + self.videoRotation = rotation + + UIView.animate(withDuration: 0.3) { + self.layoutDisplayView() + } + } + + if size != self.videoSize { + self.videoSize = size + self.onVideoSizeChanged?(size) + } + + self.displayView.sampleBufferLayer.enqueue(sampleBuffer) + } + } + + // MARK: - Frame conversion + + private static func pixelBuffer(from buffer: RTCVideoFrameBuffer) -> CVPixelBuffer? { + // Hardware decoded frames are already backed by a CVPixelBuffer + if let cvPixelBuffer = buffer as? RTCCVPixelBuffer { + return cvPixelBuffer.pixelBuffer + } + + // Software decoded frames (e.g. VP8) need to be converted to a NV12 CVPixelBuffer + return self.nv12PixelBuffer(from: buffer.toI420()) + } + + private static func nv12PixelBuffer(from i420Buffer: RTCYUVPlanarBuffer) -> CVPixelBuffer? { + let width = Int(i420Buffer.width) + let height = Int(i420Buffer.height) + let attributes: [CFString: Any] = [kCVPixelBufferIOSurfacePropertiesKey: [String: Any]()] + var pixelBufferOut: CVPixelBuffer? + + let result = CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, attributes as CFDictionary, &pixelBufferOut) + guard result == kCVReturnSuccess, let pixelBuffer = pixelBufferOut else { return nil } + + CVPixelBufferLockBaseAddress(pixelBuffer, []) + defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) } + + guard let lumaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0), + let chromaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1) + else { return nil } + + // Copy the luma plane row by row, as the strides might differ + let lumaDestination = lumaBaseAddress.assumingMemoryBound(to: UInt8.self) + let lumaDestinationStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0) + let lumaSourceStride = Int(i420Buffer.strideY) + + for row in 0.. CMSampleBuffer? { + var formatDescriptionOut: CMVideoFormatDescription? + + let formatResult = CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: pixelBuffer, formatDescriptionOut: &formatDescriptionOut) + guard formatResult == noErr, let formatDescription = formatDescriptionOut else { return nil } + + var timingInfo = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: CMClockGetTime(CMClockGetHostTimeClock()), decodeTimeStamp: .invalid) + var sampleBufferOut: CMSampleBuffer? + + let sampleResult = CMSampleBufferCreateReadyWithImageBuffer(allocator: kCFAllocatorDefault, + imageBuffer: pixelBuffer, + formatDescription: formatDescription, + sampleTiming: &timingInfo, + sampleBufferOut: &sampleBufferOut) + guard sampleResult == noErr, let sampleBuffer = sampleBufferOut else { return nil } + + // Make sure the frame is displayed as soon as it is enqueued + if let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: true), CFArrayGetCount(attachments) > 0 { + let attachment = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self) + CFDictionarySetValue(attachment, + Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(), + Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()) + } + + return sampleBuffer + } +} + +// Content view controller of the Picture in Picture window. Shows the video of the +// promoted participant, or the avatar and name when no video is available. +class CallPiPViewController: AVPictureInPictureVideoCallViewController { + + public let videoRenderView = SampleBufferVideoRenderView() + public let localVideoRenderView = SampleBufferVideoRenderView() + + private let placeholderView = UIView() + private let avatarImageView = AvatarImageView(frame: .zero) + private let displayNameLabel = UILabel() + + private var localVideoAspectConstraint: NSLayoutConstraint? + private var localVideoLandscapeSizeConstraint: NSLayoutConstraint? + private var localVideoPortraitSizeConstraint: NSLayoutConstraint? + + private let avatarSize = 80.0 + + public static let defaultContentSize = CGSize(width: 1280, height: 720) + + // The preferred content size must have its final value when the + // AVPictureInPictureController is created: AVKit determines the scale of the window + // from it well before the window is shown and ignores changes until Picture in + // Picture finished starting, so a change in between results in a wrongly scaled + // window on the very first start. Once started, size changes are applied fine + init(initialContentSize: CGSize = CallPiPViewController.defaultContentSize) { + super.init(nibName: nil, bundle: nil) + + self.preferredContentSize = initialContentSize + + // Hide here instead of in viewDidLoad: when Picture in Picture starts for the first + // time, the own video might be unhidden by the delegate before the view is loaded, + // in that case viewDidLoad would override the visibility again + self.localVideoRenderView.isHidden = true + + // The size handlers are also set up here: the first video frames can arrive before + // the view is loaded and the callback only fires when the reported size changes, + // so a closure assigned in viewDidLoad would miss the initial size + videoRenderView.onVideoSizeChanged = { [weak self] size in + self?.setVideoContentSize(size) + } + + localVideoRenderView.onVideoSizeChanged = { [weak self] size in + self?.updateLocalVideoAspectRatio(size) + } + + AllocationTracker.shared.addAllocation("CallPiPViewController") + } + + deinit { + AllocationTracker.shared.removeAllocation("CallPiPViewController") + } + + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override func viewDidLoad() { + super.viewDidLoad() + + self.view.backgroundColor = .black + + videoRenderView.translatesAutoresizingMaskIntoConstraints = false + localVideoRenderView.translatesAutoresizingMaskIntoConstraints = false + placeholderView.translatesAutoresizingMaskIntoConstraints = false + avatarImageView.translatesAutoresizingMaskIntoConstraints = false + displayNameLabel.translatesAutoresizingMaskIntoConstraints = false + + localVideoRenderView.layer.cornerRadius = 8 + localVideoRenderView.layer.masksToBounds = true + + // Match the waiting screen of the call view, where the theme color is darkened + // by a black overlay with 0.6 alpha (see AvatarBackgroundImageView) + var red: CGFloat = 0, green: CGFloat = 0, blue: CGFloat = 0 + NCAppBranding.themeColor().getRed(&red, green: &green, blue: &blue, alpha: nil) + placeholderView.backgroundColor = UIColor(red: red * 0.4, green: green * 0.4, blue: blue * 0.4, alpha: 1) + + avatarImageView.layer.masksToBounds = true + + displayNameLabel.textColor = .white + displayNameLabel.textAlignment = .center + displayNameLabel.font = .preferredFont(forTextStyle: .callout) + + placeholderView.addSubview(avatarImageView) + placeholderView.addSubview(displayNameLabel) + + self.view.addSubview(videoRenderView) + self.view.addSubview(placeholderView) + self.view.addSubview(localVideoRenderView) + + // The aspect constraint might already exist when the own video reported a size + // before the view was loaded + let localVideoAspectConstraint = self.localVideoAspectConstraint ?? localVideoRenderView.heightAnchor.constraint(equalTo: localVideoRenderView.widthAnchor, multiplier: 4.0 / 3.0) + self.localVideoAspectConstraint = localVideoAspectConstraint + + // The window aspect follows the remote video (via preferredContentSize), so size the + // own video relative to the longer side of the window, to get a consistent size for + // both portrait and landscape remote videos. The two constraints are switched in + // viewDidLayoutSubviews based on the current window size + let localVideoLandscapeSizeConstraint = localVideoRenderView.widthAnchor.constraint(equalTo: view.widthAnchor, multiplier: 0.2) + let localVideoPortraitSizeConstraint = localVideoRenderView.heightAnchor.constraint(equalTo: view.heightAnchor, multiplier: 0.2) + self.localVideoLandscapeSizeConstraint = localVideoLandscapeSizeConstraint + self.localVideoPortraitSizeConstraint = localVideoPortraitSizeConstraint + + // The avatar has a preferred fixed size, but shrinks when the window + // is too small to fit it (e.g. when it was resized by the user) + let avatarPreferredSizeConstraint = avatarImageView.heightAnchor.constraint(equalToConstant: avatarSize) + avatarPreferredSizeConstraint.priority = .defaultHigh + + NSLayoutConstraint.activate([ + localVideoAspectConstraint, + localVideoLandscapeSizeConstraint, + localVideoRenderView.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -8), + localVideoRenderView.bottomAnchor.constraint(equalTo: view.bottomAnchor, constant: -8), + + videoRenderView.topAnchor.constraint(equalTo: view.topAnchor), + videoRenderView.bottomAnchor.constraint(equalTo: view.bottomAnchor), + videoRenderView.leadingAnchor.constraint(equalTo: view.leadingAnchor), + videoRenderView.trailingAnchor.constraint(equalTo: view.trailingAnchor), + + placeholderView.topAnchor.constraint(equalTo: view.topAnchor), + placeholderView.bottomAnchor.constraint(equalTo: view.bottomAnchor), + placeholderView.leadingAnchor.constraint(equalTo: view.leadingAnchor), + placeholderView.trailingAnchor.constraint(equalTo: view.trailingAnchor), + + avatarImageView.centerXAnchor.constraint(equalTo: placeholderView.centerXAnchor), + avatarImageView.centerYAnchor.constraint(equalTo: placeholderView.centerYAnchor, constant: -24), + avatarPreferredSizeConstraint, + avatarImageView.widthAnchor.constraint(equalTo: avatarImageView.heightAnchor), + avatarImageView.heightAnchor.constraint(lessThanOrEqualTo: placeholderView.heightAnchor, multiplier: 0.4), + avatarImageView.widthAnchor.constraint(lessThanOrEqualTo: placeholderView.widthAnchor, multiplier: 0.4), + + displayNameLabel.topAnchor.constraint(equalTo: avatarImageView.bottomAnchor, constant: 8), + displayNameLabel.leadingAnchor.constraint(greaterThanOrEqualTo: placeholderView.leadingAnchor, constant: 8), + displayNameLabel.trailingAnchor.constraint(lessThanOrEqualTo: placeholderView.trailingAnchor, constant: -8), + displayNameLabel.centerXAnchor.constraint(equalTo: placeholderView.centerXAnchor) + ]) + } + + override func viewDidLayoutSubviews() { + super.viewDidLayoutSubviews() + + self.updateLocalVideoSizeConstraint() + + // Keep the avatar round, also when it is scaled down in a small window. + // At this point the nested views did not adopt their new size yet, + // so force a layout of the placeholder before reading the avatar bounds + placeholderView.layoutIfNeeded() + avatarImageView.layer.cornerRadius = avatarImageView.bounds.height / 2 + } + + private func updateLocalVideoSizeConstraint() { + guard let localVideoLandscapeSizeConstraint, let localVideoPortraitSizeConstraint else { return } + + let isWindowLandscape = view.bounds.width >= view.bounds.height + + // Always deactivate first, so both constraints are never active at the same time + if isWindowLandscape { + localVideoPortraitSizeConstraint.isActive = false + localVideoLandscapeSizeConstraint.isActive = true + } else { + localVideoLandscapeSizeConstraint.isActive = false + localVideoPortraitSizeConstraint.isActive = true + } + } + + // Match the aspect ratio of the own video view to the (rotated) size of the + // rendered video, so the video is not letterboxed inside its corner frame + private func updateLocalVideoAspectRatio(_ size: CGSize) { + guard size.width > 0, size.height > 0 else { return } + + localVideoAspectConstraint?.isActive = false + + let aspectConstraint = localVideoRenderView.heightAnchor.constraint(equalTo: localVideoRenderView.widthAnchor, multiplier: size.height / size.width) + aspectConstraint.isActive = true + + localVideoAspectConstraint = aspectConstraint + + // Animate the size change when the own video is already visible + if viewIfLoaded?.window != nil { + UIView.animate(withDuration: 0.3) { + self.view.layoutIfNeeded() + } + } + } + + // The size is normalized to a fixed dimension, so the window size only depends on the + // aspect ratio of the video and not on the resolution of the stream, which usually + // starts low and adapts shortly after the stream was established + public static func normalizedContentSize(for videoSize: CGSize) -> CGSize? { + guard videoSize.width > 0, videoSize.height > 0 else { return nil } + + let scale = 1280 / max(videoSize.width, videoSize.height) + + return CGSize(width: (videoSize.width * scale).rounded(), height: (videoSize.height * scale).rounded()) + } + + // Sets the preferred content size of the Picture in Picture window from a video size. + // Note that the window resize is performed by AVKit without an animation, there is + // no API to influence that + private func setVideoContentSize(_ size: CGSize) { + guard let normalizedSize = Self.normalizedContentSize(for: size), + normalizedSize != self.preferredContentSize + else { return } + + self.preferredContentSize = normalizedSize + } + + // Assigning an unchanged value on purpose: AVKit ignores size changes while the + // window is animating in, but reacts to the setter call once it is shown + public func reassertContentSize() { + let contentSize = self.preferredContentSize + self.preferredContentSize = contentSize + } + + public func setVideoDisabled(_ disabled: Bool) { + placeholderView.isHidden = !disabled + videoRenderView.isHidden = disabled + } + + public func setLocalVideoHidden(_ hidden: Bool) { + localVideoRenderView.isHidden = hidden + } + + public func setAvatar(for actor: TalkActor, using account: TalkAccount) { + avatarImageView.isHidden = false + avatarImageView.setActorAvatar(forId: actor.id, withType: actor.type, withDisplayName: actor.displayName, withRoomToken: nil, using: account) + displayNameLabel.text = actor.displayName + } + + public func showPlaceholder(for room: NCRoom) { + avatarImageView.isHidden = false + avatarImageView.setAvatar(for: room) + displayNameLabel.text = room.displayName + + // Without a video there's no size to follow, use the default window size + if self.preferredContentSize != Self.defaultContentSize { + self.preferredContentSize = Self.defaultContentSize + } + + self.setVideoDisabled(true) + } +} diff --git a/NextcloudTalk/Calls/CallViewController.swift b/NextcloudTalk/Calls/CallViewController.swift index 4cd33a096..c2e61cd05 100644 --- a/NextcloudTalk/Calls/CallViewController.swift +++ b/NextcloudTalk/Calls/CallViewController.swift @@ -63,6 +63,13 @@ class CallViewController: UIViewController, private let speakerLayout = CallSpeakerLayout() private var gridLayout: UICollectionViewLayout? + private var pipController: AVPictureInPictureController? + private var pipViewController: CallPiPViewController? + private var pipPeerIdentifier: String? + private weak var pipRendererAttachedPeer: NCPeerConnection? + private var pipLocalRendererAttached = false + private var isPiPActive = false + @IBOutlet public var localVideoView: MTKView! @IBOutlet public var localVideoViewWrapper: UIView! @IBOutlet public var screensharingView: NCZoomableView! @@ -307,6 +314,8 @@ class CallViewController: UIViewController, self.applyInitialSnapshot() + self.setupPictureInPicture() + self.createWaitingScreen() // We hide localVideoView until we receive it from cameraController @@ -428,7 +437,16 @@ class CallViewController: UIViewController, // MARK: - App lifecycle notifications func appDidBecomeActive(notification: NSNotification) { - if callController != nil, !isAudioOnly, !userDisabledVideo { + // Picture in Picture is only stopped automatically when the app is brought back + // with the "return to app" button of the Picture in Picture window. When returning + // via the app switcher or the app icon it needs to be stopped manually. + // This can't happen earlier (e.g. willEnterForeground), as AVKit ignores the + // stop request while the app is not active yet + if let pipController, pipController.isPictureInPictureActive { + pipController.stopPictureInPicture() + } + + if let callController, !isAudioOnly, !userDisabledVideo, !callController.isCameraUsableWhileInBackground() { // Only enabled video if it was not disabled by the user self.enableLocalVideo() } @@ -436,6 +454,10 @@ class CallViewController: UIViewController, func appWillResignActive(notification: NSNotification) { if let callController, !isAudioOnly { + // On devices that support multitasking camera access the camera keeps + // running while the app is in the background, so the video can stay enabled + guard !callController.isCameraUsableWhileInBackground() else { return } + callController.getVideoEnabledState { isEnabled in if isEnabled { // Disable video when the app moves to the background as we can't access the camera anymore. @@ -779,6 +801,212 @@ class CallViewController: UIViewController, } } + // MARK: - Picture in Picture + + func setupPictureInPicture() { + guard !isAudioOnly, AVPictureInPictureController.isPictureInPictureSupported() else { return } + + self.createPictureInPicture(withInitialContentSize: CallPiPViewController.defaultContentSize) + } + + private func createPictureInPicture(withInitialContentSize contentSize: CGSize) { + // The initial window scale is determined from the preferred content size the + // content view controller was created with (see CallPiPViewController.init), + // so both the view controller and the controller need to be recreated when the + // expected video size changes before Picture in Picture is started + self.pipController?.contentSource = nil + + let pipViewController = CallPiPViewController(initialContentSize: contentSize) + let contentSource = AVPictureInPictureController.ContentSource(activeVideoCallSourceView: self.collectionView, contentViewController: pipViewController) + let pipController = AVPictureInPictureController(contentSource: contentSource) + + // Start Picture in Picture automatically when the app is moved to the background during a call + pipController.canStartPictureInPictureAutomaticallyFromInline = true + pipController.delegate = self + + self.pipViewController = pipViewController + self.pipController = pipController + } + + // Recreate Picture in Picture when the video size of the expected initial participant + // does not match the current content size, so the window has the correct scale and + // aspect on the very first start (see createPictureInPicture). Called when a video + // size changes, but also when peers are added: a size reported while its peer was + // still a pending insert is skipped here, because initialPiPPeer can't return it yet + private func recreatePictureInPictureIfNeeded() { + guard !isPiPActive, UIApplication.shared.applicationState == .active, + let pipViewController, + let initialPeerIdentifier = initialPiPPeer()?.peerIdentifier, + let rendererView = videoRenderersDict[initialPeerIdentifier], + let expectedContentSize = CallPiPViewController.normalizedContentSize(for: rendererView.frame.size), + expectedContentSize != pipViewController.preferredContentSize + else { return } + + self.createPictureInPicture(withInitialContentSize: expectedContentSize) + } + + private func stopPictureInPicture() { + guard let pipController else { return } + + self.isPiPActive = false + self.pipPeerIdentifier = nil + self.detachPiPRenderer() + self.detachPiPLocalRenderer() + + if pipController.isPictureInPictureActive { + pipController.stopPictureInPicture() + } + + pipController.contentSource = nil + self.pipController = nil + self.pipViewController = nil + } + + private func initialPiPPeer() -> NCPeerConnection? { + // In speaker view the participant shown in Picture in Picture follows the promoted one, + // otherwise it is determined like the initially promoted participant in speaker view + if callViewMode == .speaker, let promotedPeer = peersInCall.first(where: { $0.peerIdentifier == promotedPeerIdentifier }) { + return promotedPeer + } + + return initialPromotedPeer() + } + + func promotePeerInPiP(_ peer: NCPeerConnection) { + DispatchQueue.main.async { + guard self.isPiPActive, self.pipPeerIdentifier != peer.peerIdentifier else { return } + + self.pipPeerIdentifier = peer.peerIdentifier + self.updatePiPContent() + } + } + + func promoteNextSpeakerInPiPIfNeeded(_ peer: NCPeerConnection) { + DispatchQueue.main.async { + // When the participant shown in Picture in Picture stops speaking, hand the + // promotion over to a participant that is still speaking (same as in speaker view) + guard self.isPiPActive, self.pipPeerIdentifier == peer.peerIdentifier, + let nextSpeaker = self.peersInCall.first(where: { $0.isPeerSpeaking && $0.peerIdentifier != peer.peerIdentifier }) + else { return } + + self.promotePeerInPiP(nextSpeaker) + } + } + + private func updatePiPPeerIfNeeded() { + guard isPiPActive else { return } + + // Make sure the participant shown in Picture in Picture is still in the call + if !peersInCall.contains(where: { $0.peerIdentifier == pipPeerIdentifier }) { + pipPeerIdentifier = initialPiPPeer()?.peerIdentifier + } + + self.updatePiPContent() + } + + private func updatePiPContentIfNeeded(for peer: NCPeerConnection) { + DispatchQueue.main.async { + guard self.isPiPActive, self.pipPeerIdentifier == peer.peerIdentifier else { return } + + self.updatePiPContent() + } + } + + private func reattachPiPRendererIfNeeded(for peer: NCPeerConnection) { + DispatchQueue.main.async { + guard self.isPiPActive, self.pipPeerIdentifier == peer.peerIdentifier else { return } + + // A new remote stream was added for the shown participant, + // so the renderer needs to be attached to the new video track + self.detachPiPRenderer() + self.updatePiPContent() + } + } + + private func updatePiPContent() { + guard isPiPActive, let pipViewController else { return } + + guard let peer = peersInCall.first(where: { $0.peerIdentifier == pipPeerIdentifier }) else { + if peersInCall.isEmpty { + // No other participant in the call, show the conversation avatar and name + self.detachPiPRenderer() + pipViewController.showPlaceholder(for: self.room) + } + + // Otherwise the participant was not added to the collection view yet, + // in that case the content is updated when the pending insert is processed + + return + } + + if pipRendererAttachedPeer !== peer { + self.detachPiPRenderer() + self.pipRendererAttachedPeer = peer + + WebRTCCommon.shared.dispatch { + peer.getRemoteStream()?.videoTracks.first?.add(pipViewController.videoRenderView) + } + } + + pipViewController.setVideoDisabled(peer.isRemoteVideoDisabled || !peer.hasRemoteStream()) + + WebRTCCommon.shared.dispatch { + let actor = self.callController?.getActor(fromSessionId: peer.peerId) ?? TalkActor() + + if actor.rawDisplayName.isEmpty, let peerName = peer.peerName, !peerName.isEmpty { + actor.rawDisplayName = peerName + } + + DispatchQueue.main.async { + pipViewController.setAvatar(for: actor, using: self.account) + } + } + } + + private func detachPiPRenderer() { + guard let pipViewController else { return } + + if let attachedPeer = pipRendererAttachedPeer { + WebRTCCommon.shared.dispatch { + attachedPeer.getRemoteStream()?.videoTracks.first?.remove(pipViewController.videoRenderView) + } + } + + self.pipRendererAttachedPeer = nil + pipViewController.videoRenderView.flush() + } + + private func attachPiPLocalRendererIfNeeded() { + guard isPiPActive, let callController, let pipViewController, + callController.isCameraUsableWhileInBackground() + else { return } + + self.pipLocalRendererAttached = true + + pipViewController.localVideoRenderView.isMirrored = callController.isUsingFrontCamera() + callController.attachRenderer(toLocalVideoTrack: pipViewController.localVideoRenderView) + + callController.getVideoEnabledState { isEnabled in + DispatchQueue.main.async { + guard self.pipLocalRendererAttached else { return } + + pipViewController.setLocalVideoHidden(!isEnabled) + } + } + } + + private func detachPiPLocalRenderer() { + guard let pipViewController else { return } + + if pipLocalRendererAttached { + callController?.detachRenderer(fromLocalVideoTrack: pipViewController.localVideoRenderView) + } + + self.pipLocalRendererAttached = false + pipViewController.setLocalVideoHidden(true) + pipViewController.localVideoRenderView.flush() + } + func updateParticipantCell(cell: CallParticipantViewCell, withPeerConnection peerConnection: NCPeerConnection) { var isVideoDisabled = peerConnection.isRemoteVideoDisabled @@ -919,6 +1147,15 @@ class CallViewController: UIViewController, self.userDisabledVideo = true } + DispatchQueue.main.async { + // The local video track might have been recreated while in Picture in Picture + // (e.g. when reconnecting a call), so the renderer needs to be attached to the new track + if self.pipLocalRendererAttached { + self.detachPiPLocalRenderer() + self.attachPiPLocalRendererIfNeeded() + } + } + #if targetEnvironment(simulator) // On the simulator there's no camera controller drawing on localVideoView, // so we render the generated local track with an RTCMTLVideoView instead @@ -970,6 +1207,8 @@ class CallViewController: UIViewController, // This is a new peer, add it self.addPeer(remotePeer) } + + self.reattachPiPRendererIfNeeded(for: remotePeer) } else if remotePeer.roomType == kRoomTypeScreen { self.screenRenderersDict[remotePeer.peerId] = renderView self.screenPeersInCall.append(remotePeer) @@ -1020,6 +1259,8 @@ class CallViewController: UIViewController, self.updatePeer(peer) { cell in cell.videoDisabled = peer.isRemoteVideoDisabled } + + self.updatePiPContentIfNeeded(for: peer) } case "speaking", "stoppedSpeaking": if peersInCall.count > 1 { @@ -1029,9 +1270,11 @@ class CallViewController: UIViewController, // Add to speakers and sort participants if needed if message == "speaking" { self.promotePeerInSpeakerView(peer) + self.promotePeerInPiP(peer) self.addSpeakerAndPromoteIfNeeded(peer) } else { self.promoteNextSpeakerInSpeakerViewIfNeeded(peer) + self.promoteNextSpeakerInPiPIfNeeded(peer) } } case "raiseHand": @@ -1048,6 +1291,8 @@ class CallViewController: UIViewController, cell.displayName = nick } + self.updatePiPContentIfNeeded(for: peer) + if peer.peerId == self.presentedScreenPeerId { DispatchQueue.main.async { self.screenshareLabel.text = nick @@ -1970,6 +2215,12 @@ class CallViewController: UIViewController, self.setLocalVideoViewWrapperHidden(!enabled) self.setVideoDisableButtonActive(enabled) + + DispatchQueue.main.async { + if self.pipLocalRendererAttached { + self.pipViewController?.setLocalVideoHidden(!enabled) + } + } } @IBAction func switchCameraButtonPressed(_ sender: Any) { @@ -2064,6 +2315,8 @@ class CallViewController: UIViewController, self.moreMenuButton?.contextMenuInteraction?.dismissMenu() self.hangUpButton?.contextMenuInteraction?.dismissMenu() + self.stopPictureInPicture() + self.delegate?.callViewControllerWantsToBeDismissed(self) callController?.stopCapturing() @@ -2384,6 +2637,10 @@ class CallViewController: UIViewController, func finishCall() { callController = nil + DispatchQueue.main.async { + self.stopPictureInPicture() + } + if videoCallUpgrade { videoCallUpgrade = false self.delegate?.callViewControllerWantsVideoCallUpgrade(self) @@ -2570,6 +2827,8 @@ class CallViewController: UIViewController, participantCell.setRemoteVideoSize(size) } + + self.recreatePictureInPictureIfNeeded() } for (_, rendererView) in self.screenRenderersDict.filter({ $0.value == mtlVideoView }) { @@ -2650,6 +2909,8 @@ class CallViewController: UIViewController, // Don't delay adding the first peer self.peersInCall.append(peer) self.updateSnapshot() + self.updatePiPPeerIfNeeded() + self.recreatePictureInPictureIfNeeded() } else { // Delay updating the collection view a bit to allow batch updating self.pendingPeerInserts.append(peer) @@ -2725,6 +2986,9 @@ class CallViewController: UIViewController, promotedPeerIdentifier = initialPromotedPeer()?.peerIdentifier } + self.updatePiPPeerIfNeeded() + self.recreatePictureInPictureIfNeeded() + // Sort peers in call self.sortPeersInCall() @@ -2772,3 +3036,50 @@ class CallViewController: UIViewController, showRoomInfo() } } + +// MARK: - AVPictureInPictureControllerDelegate + +extension CallViewController: AVPictureInPictureControllerDelegate { + + func pictureInPictureControllerWillStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { + self.isPiPActive = true + self.pipPeerIdentifier = self.initialPiPPeer()?.peerIdentifier + self.updatePiPContent() + self.attachPiPLocalRendererIfNeeded() + } + + func pictureInPictureControllerDidStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { + // When quickly switching back to the app while Picture in Picture is still + // animating in, the stop in appDidBecomeActive is missed, because the + // controller was not marked as active there yet. Stop it here instead + if UIApplication.shared.applicationState == .active { + pictureInPictureController.stopPictureInPicture() + return + } + + // A size change that happened while the window was animating in was ignored + // by AVKit, so assert the current size again now that the window is shown + self.pipViewController?.reassertContentSize() + } + + func pictureInPictureControllerDidStopPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { + self.isPiPActive = false + self.pipPeerIdentifier = nil + self.detachPiPRenderer() + self.detachPiPLocalRenderer() + } + + func pictureInPictureController(_ pictureInPictureController: AVPictureInPictureController, failedToStartPictureInPictureWithError error: Error) { + print("Failed to start Picture in Picture: \(error)") + + self.isPiPActive = false + self.pipPeerIdentifier = nil + self.detachPiPRenderer() + self.detachPiPLocalRenderer() + } + + func pictureInPictureController(_ pictureInPictureController: AVPictureInPictureController, restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void) { + // The call view is still presented fullscreen, nothing to restore + completionHandler(true) + } +} diff --git a/NextcloudTalk/Calls/NCCallController.swift b/NextcloudTalk/Calls/NCCallController.swift index 9c8343bc9..fc9bf773d 100644 --- a/NextcloudTalk/Calls/NCCallController.swift +++ b/NextcloudTalk/Calls/NCCallController.swift @@ -116,6 +116,7 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling private var localVideoTrack: RTCVideoTrack? private var localScreenTrack: RTCVideoTrack? private var localVideoCaptureController: ARDCaptureController? + private var videoDisabledDueToInterruption = false private let screensharingController = NCScreensharingController() @@ -573,6 +574,31 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling } } } + + public func isUsingFrontCamera() -> Bool { + return self.cameraController?.isUsingFrontCamera() ?? false + } + + public func isCameraUsableWhileInBackground() -> Bool { + #if targetEnvironment(simulator) + // The generated test pattern keeps running while the app is in the background + return self.simulatorVideoCapturer != nil + #else + return self.cameraController?.isMultitaskingCameraAccessEnabled() ?? false + #endif + } + + public func attachRenderer(toLocalVideoTrack renderer: RTCVideoRenderer) { + WebRTCCommon.shared.dispatch { + self.localVideoTrack?.add(renderer) + } + } + + public func detachRenderer(fromLocalVideoTrack renderer: RTCVideoRenderer) { + WebRTCCommon.shared.dispatch { + self.localVideoTrack?.remove(renderer) + } + } public func enableVideo(_ enable: Bool) { WebRTCCommon.shared.dispatch { @@ -1471,6 +1497,30 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling self.delegate?.callControllerDidDrawFirstLocalFrame(self) } + func cameraSessionWasInterrupted() { + WebRTCCommon.shared.dispatch { + // When the capture session gets interrupted (e.g. while the Picture in Picture + // window is stashed to the side of the screen), let the other participants + // know that there's no video, so they show the avatar instead of a frozen frame + guard self.isVideoEnabled() else { return } + + self.videoDisabledDueToInterruption = true + self.localVideoTrack?.isEnabled = false + self.sendMessageToAll(ofType: "videoOff", withPayload: nil) + } + } + + func cameraSessionInterruptionEnded() { + WebRTCCommon.shared.dispatch { + // Only enable the video again if it was disabled because of the interruption + guard self.videoDisabledDueToInterruption else { return } + + self.videoDisabledDueToInterruption = false + self.localVideoTrack?.isEnabled = true + self.sendMessageToAll(ofType: "videoOn", withPayload: nil) + } + } + // MARK: - NCPeerConnection delegate // Delegates from NCPeerConnection are already dispatched to the webrtc worker queue diff --git a/NextcloudTalk/Calls/NCCameraController.swift b/NextcloudTalk/Calls/NCCameraController.swift index 070e25947..b04dc6003 100644 --- a/NextcloudTalk/Calls/NCCameraController.swift +++ b/NextcloudTalk/Calls/NCCameraController.swift @@ -12,6 +12,8 @@ import MetalKit @objc protocol NCCameraControllerDelegate { @objc func didDrawFirstFrameOnLocalView() + @objc func cameraSessionWasInterrupted() + @objc func cameraSessionInterruptionEnded() } @objcMembers @@ -71,6 +73,9 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate initAVCaptureSession() NotificationCenter.default.addObserver(self, selector: #selector(deviceOrientationDidChangeNotification), name: UIDevice.orientationDidChangeNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(sessionWasInterrupted(notification:)), name: AVCaptureSession.wasInterruptedNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(sessionInterruptionEnded(notification:)), name: AVCaptureSession.interruptionEndedNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(sessionRuntimeError(notification:)), name: AVCaptureSession.runtimeErrorNotification, object: nil) self.updateVideoRotationBasedOnDeviceOrientation() } @@ -206,6 +211,12 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate self.session?.sessionPreset = .inputPriority + // Keep the camera running while the app is in the background during + // Picture in Picture on devices that support multitasking camera access + if self.session?.isMultitaskingCameraAccessSupported ?? false { + self.session?.isMultitaskingCameraAccessEnabled = true + } + if let input = self.usingFrontCamera ? self.getFrontCameraInput() : self.getBackCameraInput() { self.session?.addInput(input) } @@ -224,6 +235,14 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate self.session?.stopRunning() } + public func isMultitaskingCameraAccessEnabled() -> Bool { + return session?.isMultitaskingCameraAccessEnabled ?? false + } + + public func isUsingFrontCamera() -> Bool { + return usingFrontCamera + } + // MARK: - Public switches public func enableBackgroundBlur(enable: Bool) { @@ -389,6 +408,47 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate // MARK: - Notifications + func sessionWasInterrupted(notification: Notification) { + guard let session = notification.object as? AVCaptureSession, session === self.session else { return } + + // The system interrupts the capture session when the camera is not available + // anymore, e.g. when the Picture in Picture window is stashed to the side of + // the screen while using the camera in the background + if let reason = notification.userInfo?[AVCaptureSessionInterruptionReasonKey] as? Int { + print("Capture session was interrupted, reason \(reason)") + } + + delegate?.cameraSessionWasInterrupted() + } + + func sessionInterruptionEnded(notification: Notification) { + guard let session = notification.object as? AVCaptureSession, session === self.session else { return } + + print("Capture session interruption ended") + + // The session should resume on its own when the interruption ended, + // but make sure it is really running again + DispatchQueue.global(qos: .userInitiated).async { [weak self] in + guard let session = self?.session, !session.isRunning else { return } + + session.startRunning() + } + + delegate?.cameraSessionInterruptionEnded() + } + + func sessionRuntimeError(notification: Notification) { + guard let session = notification.object as? AVCaptureSession, session === self.session else { return } + + let error = notification.userInfo?[AVCaptureSessionErrorKey] as? NSError + print("Capture session runtime error: \(String(describing: error))") + + // Try to restart the session after a runtime error + DispatchQueue.global(qos: .userInitiated).async { [weak self] in + self?.session?.startRunning() + } + } + func deviceOrientationDidChangeNotification() { let currentOrientation = UIDevice.current.orientation diff --git a/NextcloudTalk/Calls/SimulatorVideoCapturer.swift b/NextcloudTalk/Calls/SimulatorVideoCapturer.swift index bd7c08106..1f0b729ce 100644 --- a/NextcloudTalk/Calls/SimulatorVideoCapturer.swift +++ b/NextcloudTalk/Calls/SimulatorVideoCapturer.swift @@ -11,7 +11,8 @@ import WebRTC /// Video capturer for the simulator, where no camera is available. /// Generates a test pattern (color slowly cycling through the hue spectrum), -/// so video can still be debugged on the simulator. +/// so video can still be debugged on the simulator. The frames are rotated +/// based on the simulated device orientation, like a real camera would do. class SimulatorVideoCapturer: RTCVideoCapturer { private static let frameWidth = 640 @@ -24,12 +25,20 @@ class SimulatorVideoCapturer: RTCVideoCapturer { private let captureQueue = DispatchQueue(label: "simulatorvideocapturer") private var timer: DispatchSourceTimer? private var frameNumber = 0 + private var videoRotation = RTCVideoRotation._90 deinit { timer?.cancel() + NotificationCenter.default.removeObserver(self) } public func startCapture() { + DispatchQueue.main.async { + // Orientation notifications are already generated app-wide, see AppDelegate + NotificationCenter.default.addObserver(self, selector: #selector(self.deviceOrientationDidChangeNotification), name: UIDevice.orientationDidChangeNotification, object: nil) + self.updateVideoRotation() + } + captureQueue.async { guard self.timer == nil else { return } @@ -44,6 +53,32 @@ class SimulatorVideoCapturer: RTCVideoCapturer { } } + @objc private func deviceOrientationDidChangeNotification() { + self.updateVideoRotation() + } + + private func updateVideoRotation() { + let rotation: RTCVideoRotation + + switch UIDevice.current.orientation { + case .portrait: + rotation = ._90 + case .portraitUpsideDown: + rotation = ._270 + case .landscapeLeft: + rotation = ._0 + case .landscapeRight: + rotation = ._180 + default: + // Keep the current rotation for face up/down and unknown orientations + return + } + + captureQueue.async { + self.videoRotation = rotation + } + } + public func stopCapture() { captureQueue.async { self.timer?.cancel() @@ -63,7 +98,7 @@ class SimulatorVideoCapturer: RTCVideoCapturer { let timeStampNs = CACurrentMediaTime() * Float64(NSEC_PER_SEC) let rtcPixelBuffer = RTCCVPixelBuffer(pixelBuffer: pixelBuffer) - let videoFrame = RTCVideoFrame(buffer: rtcPixelBuffer, rotation: ._0, timeStampNs: Int64(timeStampNs)) + let videoFrame = RTCVideoFrame(buffer: rtcPixelBuffer, rotation: videoRotation, timeStampNs: Int64(timeStampNs)) self.delegate?.capturer(self, didCapture: videoFrame) }