From c98d9dc16d03e41f834d9579f34cbdd2b6c547a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Thu, 16 Jul 2026 18:38:16 +0200 Subject: [PATCH 1/8] fix: Simulator should send video rotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcel Müller --- .../Calls/SimulatorVideoCapturer.swift | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/NextcloudTalk/Calls/SimulatorVideoCapturer.swift b/NextcloudTalk/Calls/SimulatorVideoCapturer.swift index bd7c08106..1f0b729ce 100644 --- a/NextcloudTalk/Calls/SimulatorVideoCapturer.swift +++ b/NextcloudTalk/Calls/SimulatorVideoCapturer.swift @@ -11,7 +11,8 @@ import WebRTC /// Video capturer for the simulator, where no camera is available. /// Generates a test pattern (color slowly cycling through the hue spectrum), -/// so video can still be debugged on the simulator. +/// so video can still be debugged on the simulator. The frames are rotated +/// based on the simulated device orientation, like a real camera would do. class SimulatorVideoCapturer: RTCVideoCapturer { private static let frameWidth = 640 @@ -24,12 +25,20 @@ class SimulatorVideoCapturer: RTCVideoCapturer { private let captureQueue = DispatchQueue(label: "simulatorvideocapturer") private var timer: DispatchSourceTimer? private var frameNumber = 0 + private var videoRotation = RTCVideoRotation._90 deinit { timer?.cancel() + NotificationCenter.default.removeObserver(self) } public func startCapture() { + DispatchQueue.main.async { + // Orientation notifications are already generated app-wide, see AppDelegate + NotificationCenter.default.addObserver(self, selector: #selector(self.deviceOrientationDidChangeNotification), name: UIDevice.orientationDidChangeNotification, object: nil) + self.updateVideoRotation() + } + captureQueue.async { guard self.timer == nil else { return } @@ -44,6 +53,32 @@ class SimulatorVideoCapturer: RTCVideoCapturer { } } + @objc private func deviceOrientationDidChangeNotification() { + self.updateVideoRotation() + } + + private func updateVideoRotation() { + let rotation: RTCVideoRotation + + switch UIDevice.current.orientation { + case .portrait: + rotation = ._90 + case .portraitUpsideDown: + rotation = ._270 + case .landscapeLeft: + rotation = ._0 + case .landscapeRight: + rotation = ._180 + default: + // Keep the current rotation for face up/down and unknown orientations + return + } + + captureQueue.async { + self.videoRotation = rotation + } + } + public func stopCapture() { captureQueue.async { self.timer?.cancel() @@ -63,7 +98,7 @@ class SimulatorVideoCapturer: RTCVideoCapturer { let timeStampNs = CACurrentMediaTime() * Float64(NSEC_PER_SEC) let rtcPixelBuffer = RTCCVPixelBuffer(pixelBuffer: pixelBuffer) - let videoFrame = RTCVideoFrame(buffer: rtcPixelBuffer, rotation: ._0, timeStampNs: Int64(timeStampNs)) + let videoFrame = RTCVideoFrame(buffer: rtcPixelBuffer, rotation: videoRotation, timeStampNs: Int64(timeStampNs)) self.delegate?.capturer(self, didCapture: videoFrame) } From a17f209f77dd7dc8d83dc64fdf730a913a99f8f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Thu, 16 Jul 2026 18:56:36 +0200 Subject: [PATCH 2/8] feat: Initial picture in picture support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 Signed-off-by: Marcel Müller --- .../Calls/CallPiPViewController.swift | 429 ++++++++++++++++++ NextcloudTalk/Calls/CallViewController.swift | 288 +++++++++++- NextcloudTalk/Calls/NCCallController.swift | 25 + NextcloudTalk/Calls/NCCameraController.swift | 14 + 4 files changed, 755 insertions(+), 1 deletion(-) create mode 100644 NextcloudTalk/Calls/CallPiPViewController.swift diff --git a/NextcloudTalk/Calls/CallPiPViewController.swift b/NextcloudTalk/Calls/CallPiPViewController.swift new file mode 100644 index 000000000..4b2aca197 --- /dev/null +++ b/NextcloudTalk/Calls/CallPiPViewController.swift @@ -0,0 +1,429 @@ +// +// SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors +// SPDX-License-Identifier: GPL-3.0-or-later +// + +import AVKit +import Foundation +import UIKit +import WebRTC + +// Renders remote RTCVideoFrames into an AVSampleBufferDisplayLayer. +// Metal (RTCMTLVideoView) is not allowed to render while the app is in the background, +// AVSampleBufferDisplayLayer is, which makes it the only way to show remote video +// in the Picture in Picture window. +class SampleBufferVideoRenderView: UIView, RTCVideoRenderer { + + private class DisplayLayerView: UIView { + override class var layerClass: AnyClass { + return AVSampleBufferDisplayLayer.self + } + + var sampleBufferLayer: AVSampleBufferDisplayLayer { + // swiftlint:disable:next force_cast + return layer as! AVSampleBufferDisplayLayer + } + } + + // Called on the main queue whenever the (rotated) size of the rendered video changes + public var onVideoSizeChanged: ((CGSize) -> Void)? + + // Mirrors the rendered video, as it is expected for the own video of the front camera + public var isMirrored = false { + didSet { + guard isMirrored != oldValue else { return } + + DispatchQueue.main.async { + self.transform = self.isMirrored ? CGAffineTransform(scaleX: -1, y: 1) : .identity + } + } + } + + private let displayView = DisplayLayerView() + private var videoRotation = RTCVideoRotation._0 + + // The (rotated) size of the currently rendered video + public private(set) var videoSize: CGSize = .zero + + override init(frame: CGRect) { + super.init(frame: frame) + + displayView.sampleBufferLayer.videoGravity = .resizeAspect + self.addSubview(displayView) + } + + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override func layoutSubviews() { + super.layoutSubviews() + self.layoutDisplayView() + } + + // The frames of the sample buffer layer are not rotated, so the rotation + // of the video frames is applied as a transform on the layer's view + private func layoutDisplayView() { + let isRotatedSideways = videoRotation == ._90 || videoRotation == ._270 + + displayView.center = CGPoint(x: bounds.midX, y: bounds.midY) + displayView.bounds = isRotatedSideways ? CGRect(x: 0, y: 0, width: bounds.height, height: bounds.width) : bounds + + switch videoRotation { + case ._90: + displayView.transform = CGAffineTransform(rotationAngle: .pi / 2) + case ._180: + displayView.transform = CGAffineTransform(rotationAngle: .pi) + case ._270: + displayView.transform = CGAffineTransform(rotationAngle: -.pi / 2) + default: + displayView.transform = .identity + } + } + + public func flush() { + DispatchQueue.main.async { + self.displayView.sampleBufferLayer.flushAndRemoveImage() + } + } + + // MARK: - RTCVideoRenderer + + public func setSize(_ size: CGSize) { + // The size reported here does not include the frame rotation, + // so the rotated size is determined in renderFrame instead + } + + public func renderFrame(_ frame: RTCVideoFrame?) { + // Called on the WebRTC decoding thread + guard let frame, + let pixelBuffer = Self.pixelBuffer(from: frame.buffer), + let sampleBuffer = Self.sampleBuffer(from: pixelBuffer) + else { return } + + let rotation = frame.rotation + var size = CGSize(width: CGFloat(frame.width), height: CGFloat(frame.height)) + + if rotation == ._90 || rotation == ._270 { + size = CGSize(width: size.height, height: size.width) + } + + DispatchQueue.main.async { + if self.displayView.sampleBufferLayer.status == .failed { + self.displayView.sampleBufferLayer.flush() + } + + if rotation != self.videoRotation { + self.videoRotation = rotation + + UIView.animate(withDuration: 0.3) { + self.layoutDisplayView() + } + } + + if size != self.videoSize { + self.videoSize = size + self.onVideoSizeChanged?(size) + } + + self.displayView.sampleBufferLayer.enqueue(sampleBuffer) + } + } + + // MARK: - Frame conversion + + private static func pixelBuffer(from buffer: RTCVideoFrameBuffer) -> CVPixelBuffer? { + // Hardware decoded frames are already backed by a CVPixelBuffer + if let cvPixelBuffer = buffer as? RTCCVPixelBuffer { + return cvPixelBuffer.pixelBuffer + } + + // Software decoded frames (e.g. VP8) need to be converted to a NV12 CVPixelBuffer + return self.nv12PixelBuffer(from: buffer.toI420()) + } + + private static func nv12PixelBuffer(from i420Buffer: RTCYUVPlanarBuffer) -> CVPixelBuffer? { + let width = Int(i420Buffer.width) + let height = Int(i420Buffer.height) + let attributes: [CFString: Any] = [kCVPixelBufferIOSurfacePropertiesKey: [String: Any]()] + var pixelBufferOut: CVPixelBuffer? + + let result = CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, attributes as CFDictionary, &pixelBufferOut) + guard result == kCVReturnSuccess, let pixelBuffer = pixelBufferOut else { return nil } + + CVPixelBufferLockBaseAddress(pixelBuffer, []) + defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) } + + guard let lumaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0), + let chromaBaseAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1) + else { return nil } + + // Copy the luma plane row by row, as the strides might differ + let lumaDestination = lumaBaseAddress.assumingMemoryBound(to: UInt8.self) + let lumaDestinationStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0) + let lumaSourceStride = Int(i420Buffer.strideY) + + for row in 0.. CMSampleBuffer? { + var formatDescriptionOut: CMVideoFormatDescription? + + let formatResult = CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: pixelBuffer, formatDescriptionOut: &formatDescriptionOut) + guard formatResult == noErr, let formatDescription = formatDescriptionOut else { return nil } + + var timingInfo = CMSampleTimingInfo(duration: .invalid, presentationTimeStamp: CMClockGetTime(CMClockGetHostTimeClock()), decodeTimeStamp: .invalid) + var sampleBufferOut: CMSampleBuffer? + + let sampleResult = CMSampleBufferCreateReadyWithImageBuffer(allocator: kCFAllocatorDefault, + imageBuffer: pixelBuffer, + formatDescription: formatDescription, + sampleTiming: &timingInfo, + sampleBufferOut: &sampleBufferOut) + guard sampleResult == noErr, let sampleBuffer = sampleBufferOut else { return nil } + + // Make sure the frame is displayed as soon as it is enqueued + if let attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, createIfNecessary: true), CFArrayGetCount(attachments) > 0 { + let attachment = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self) + CFDictionarySetValue(attachment, + Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(), + Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()) + } + + return sampleBuffer + } +} + +// Content view controller of the Picture in Picture window. Shows the video of the +// promoted participant, or the avatar and name when no video is available. +class CallPiPViewController: AVPictureInPictureVideoCallViewController { + + public let videoRenderView = SampleBufferVideoRenderView() + public let localVideoRenderView = SampleBufferVideoRenderView() + + private let placeholderView = UIView() + private let avatarImageView = AvatarImageView(frame: .zero) + private let displayNameLabel = UILabel() + + private var localVideoAspectConstraint: NSLayoutConstraint? + private var localVideoLandscapeSizeConstraint: NSLayoutConstraint? + private var localVideoPortraitSizeConstraint: NSLayoutConstraint? + + private let avatarSize = 80.0 + + init() { + super.init(nibName: nil, bundle: nil) + + self.preferredContentSize = CGSize(width: 1280, height: 720) + + // Hide here instead of in viewDidLoad: when Picture in Picture starts for the first + // time, the own video might be unhidden by the delegate before the view is loaded, + // in that case viewDidLoad would override the visibility again + self.localVideoRenderView.isHidden = true + + // The size handlers are also set up here: the first video frames can arrive before + // the view is loaded and the callback only fires when the reported size changes, + // so a closure assigned in viewDidLoad would miss the initial size + videoRenderView.onVideoSizeChanged = { [weak self] size in + self?.setVideoContentSize(size) + } + + localVideoRenderView.onVideoSizeChanged = { [weak self] size in + self?.updateLocalVideoAspectRatio(size) + } + + AllocationTracker.shared.addAllocation("CallPiPViewController") + } + + deinit { + AllocationTracker.shared.removeAllocation("CallPiPViewController") + } + + required init?(coder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + + override func viewDidLoad() { + super.viewDidLoad() + + self.view.backgroundColor = .black + + videoRenderView.translatesAutoresizingMaskIntoConstraints = false + localVideoRenderView.translatesAutoresizingMaskIntoConstraints = false + placeholderView.translatesAutoresizingMaskIntoConstraints = false + avatarImageView.translatesAutoresizingMaskIntoConstraints = false + displayNameLabel.translatesAutoresizingMaskIntoConstraints = false + + localVideoRenderView.layer.cornerRadius = 8 + localVideoRenderView.layer.masksToBounds = true + + avatarImageView.layer.cornerRadius = avatarSize / 2 + avatarImageView.layer.masksToBounds = true + + displayNameLabel.textColor = .white + displayNameLabel.textAlignment = .center + displayNameLabel.font = .preferredFont(forTextStyle: .callout) + + placeholderView.addSubview(avatarImageView) + placeholderView.addSubview(displayNameLabel) + + self.view.addSubview(videoRenderView) + self.view.addSubview(placeholderView) + self.view.addSubview(localVideoRenderView) + + // The aspect constraint might already exist when the own video reported a size + // before the view was loaded + let localVideoAspectConstraint = self.localVideoAspectConstraint ?? localVideoRenderView.heightAnchor.constraint(equalTo: localVideoRenderView.widthAnchor, multiplier: 4.0 / 3.0) + self.localVideoAspectConstraint = localVideoAspectConstraint + + // The window aspect follows the remote video (via preferredContentSize), so size the + // own video relative to the longer side of the window, to get a consistent size for + // both portrait and landscape remote videos. The two constraints are switched in + // viewDidLayoutSubviews based on the current window size + let localVideoLandscapeSizeConstraint = localVideoRenderView.widthAnchor.constraint(equalTo: view.widthAnchor, multiplier: 0.2) + let localVideoPortraitSizeConstraint = localVideoRenderView.heightAnchor.constraint(equalTo: view.heightAnchor, multiplier: 0.2) + self.localVideoLandscapeSizeConstraint = localVideoLandscapeSizeConstraint + self.localVideoPortraitSizeConstraint = localVideoPortraitSizeConstraint + + NSLayoutConstraint.activate([ + localVideoAspectConstraint, + localVideoLandscapeSizeConstraint, + localVideoRenderView.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -8), + localVideoRenderView.bottomAnchor.constraint(equalTo: view.bottomAnchor, constant: -8), + + videoRenderView.topAnchor.constraint(equalTo: view.topAnchor), + videoRenderView.bottomAnchor.constraint(equalTo: view.bottomAnchor), + videoRenderView.leadingAnchor.constraint(equalTo: view.leadingAnchor), + videoRenderView.trailingAnchor.constraint(equalTo: view.trailingAnchor), + + placeholderView.topAnchor.constraint(equalTo: view.topAnchor), + placeholderView.bottomAnchor.constraint(equalTo: view.bottomAnchor), + placeholderView.leadingAnchor.constraint(equalTo: view.leadingAnchor), + placeholderView.trailingAnchor.constraint(equalTo: view.trailingAnchor), + + avatarImageView.centerXAnchor.constraint(equalTo: placeholderView.centerXAnchor), + avatarImageView.centerYAnchor.constraint(equalTo: placeholderView.centerYAnchor, constant: -12), + avatarImageView.widthAnchor.constraint(equalToConstant: avatarSize), + avatarImageView.heightAnchor.constraint(equalToConstant: avatarSize), + + displayNameLabel.topAnchor.constraint(equalTo: avatarImageView.bottomAnchor, constant: 8), + displayNameLabel.leadingAnchor.constraint(greaterThanOrEqualTo: placeholderView.leadingAnchor, constant: 8), + displayNameLabel.trailingAnchor.constraint(lessThanOrEqualTo: placeholderView.trailingAnchor, constant: -8), + displayNameLabel.centerXAnchor.constraint(equalTo: placeholderView.centerXAnchor) + ]) + } + + override func viewDidLayoutSubviews() { + super.viewDidLayoutSubviews() + self.updateLocalVideoSizeConstraint() + } + + private func updateLocalVideoSizeConstraint() { + guard let localVideoLandscapeSizeConstraint, let localVideoPortraitSizeConstraint else { return } + + let isWindowLandscape = view.bounds.width >= view.bounds.height + + // Always deactivate first, so both constraints are never active at the same time + if isWindowLandscape { + localVideoPortraitSizeConstraint.isActive = false + localVideoLandscapeSizeConstraint.isActive = true + } else { + localVideoLandscapeSizeConstraint.isActive = false + localVideoPortraitSizeConstraint.isActive = true + } + } + + // Match the aspect ratio of the own video view to the (rotated) size of the + // rendered video, so the video is not letterboxed inside its corner frame + private func updateLocalVideoAspectRatio(_ size: CGSize) { + guard size.width > 0, size.height > 0 else { return } + + localVideoAspectConstraint?.isActive = false + + let aspectConstraint = localVideoRenderView.heightAnchor.constraint(equalTo: localVideoRenderView.widthAnchor, multiplier: size.height / size.width) + aspectConstraint.isActive = true + + localVideoAspectConstraint = aspectConstraint + + // Animate the size change when the own video is already visible + if viewIfLoaded?.window != nil { + UIView.animate(withDuration: 0.3) { + self.view.layoutIfNeeded() + } + } + } + + // The size is normalized to a fixed dimension, so the window size only depends on the + // aspect ratio of the video and not on the resolution of the stream, which usually + // starts low and adapts shortly after the stream was established + private static func normalizedContentSize(for videoSize: CGSize) -> CGSize? { + guard videoSize.width > 0, videoSize.height > 0 else { return nil } + + let scale = 1280 / max(videoSize.width, videoSize.height) + + return CGSize(width: (videoSize.width * scale).rounded(), height: (videoSize.height * scale).rounded()) + } + + // Sets the preferred content size of the Picture in Picture window from a video size. + // Note that the window resize is performed by AVKit without an animation, there is + // no API to influence that + public func setVideoContentSize(_ size: CGSize) { + guard let normalizedSize = Self.normalizedContentSize(for: size), + normalizedSize != self.preferredContentSize + else { return } + + self.preferredContentSize = normalizedSize + } + + // Re-assert the size of the currently rendered video: AVKit ignores changes to + // preferredContentSize while the Picture in Picture window is still animating in, + // so the setter needs to be called again even though the value did not change + public func reassertVideoContentSize() { + guard let normalizedSize = Self.normalizedContentSize(for: videoRenderView.videoSize) else { return } + + self.preferredContentSize = normalizedSize + } + + public func setVideoDisabled(_ disabled: Bool) { + placeholderView.isHidden = !disabled + videoRenderView.isHidden = disabled + } + + public func setLocalVideoHidden(_ hidden: Bool) { + localVideoRenderView.isHidden = hidden + } + + public func setAvatar(for actor: TalkActor, using account: TalkAccount) { + avatarImageView.isHidden = false + avatarImageView.setActorAvatar(forId: actor.id, withType: actor.type, withDisplayName: actor.displayName, withRoomToken: nil, using: account) + displayNameLabel.text = actor.displayName + } + + public func showPlaceholder(withDisplayName displayName: String?) { + avatarImageView.isHidden = true + displayNameLabel.text = displayName + + self.setVideoDisabled(true) + } +} diff --git a/NextcloudTalk/Calls/CallViewController.swift b/NextcloudTalk/Calls/CallViewController.swift index 4cd33a096..840b937d2 100644 --- a/NextcloudTalk/Calls/CallViewController.swift +++ b/NextcloudTalk/Calls/CallViewController.swift @@ -63,6 +63,13 @@ class CallViewController: UIViewController, private let speakerLayout = CallSpeakerLayout() private var gridLayout: UICollectionViewLayout? + private var pipController: AVPictureInPictureController? + private var pipViewController: CallPiPViewController? + private var pipPeerIdentifier: String? + private weak var pipRendererAttachedPeer: NCPeerConnection? + private var pipLocalRendererAttached = false + private var isPiPActive = false + @IBOutlet public var localVideoView: MTKView! @IBOutlet public var localVideoViewWrapper: UIView! @IBOutlet public var screensharingView: NCZoomableView! @@ -307,6 +314,8 @@ class CallViewController: UIViewController, self.applyInitialSnapshot() + self.setupPictureInPicture() + self.createWaitingScreen() // We hide localVideoView until we receive it from cameraController @@ -428,7 +437,16 @@ class CallViewController: UIViewController, // MARK: - App lifecycle notifications func appDidBecomeActive(notification: NSNotification) { - if callController != nil, !isAudioOnly, !userDisabledVideo { + // Picture in Picture is only stopped automatically when the app is brought back + // with the "return to app" button of the Picture in Picture window. When returning + // via the app switcher or the app icon it needs to be stopped manually. + // This can't happen earlier (e.g. willEnterForeground), as AVKit ignores the + // stop request while the app is not active yet + if let pipController, pipController.isPictureInPictureActive { + pipController.stopPictureInPicture() + } + + if let callController, !isAudioOnly, !userDisabledVideo, !callController.isCameraUsableWhileInBackground() { // Only enabled video if it was not disabled by the user self.enableLocalVideo() } @@ -436,6 +454,10 @@ class CallViewController: UIViewController, func appWillResignActive(notification: NSNotification) { if let callController, !isAudioOnly { + // On devices that support multitasking camera access the camera keeps + // running while the app is in the background, so the video can stay enabled + guard !callController.isCameraUsableWhileInBackground() else { return } + callController.getVideoEnabledState { isEnabled in if isEnabled { // Disable video when the app moves to the background as we can't access the camera anymore. @@ -779,6 +801,196 @@ class CallViewController: UIViewController, } } + // MARK: - Picture in Picture + + func setupPictureInPicture() { + guard !isAudioOnly, AVPictureInPictureController.isPictureInPictureSupported() else { return } + + let pipViewController = CallPiPViewController() + let contentSource = AVPictureInPictureController.ContentSource(activeVideoCallSourceView: self.collectionView, contentViewController: pipViewController) + let pipController = AVPictureInPictureController(contentSource: contentSource) + + // Start Picture in Picture automatically when the app is moved to the background during a call + pipController.canStartPictureInPictureAutomaticallyFromInline = true + pipController.delegate = self + + self.pipViewController = pipViewController + self.pipController = pipController + } + + private func stopPictureInPicture() { + guard let pipController else { return } + + self.isPiPActive = false + self.pipPeerIdentifier = nil + self.detachPiPRenderer() + self.detachPiPLocalRenderer() + + if pipController.isPictureInPictureActive { + pipController.stopPictureInPicture() + } + + pipController.contentSource = nil + self.pipController = nil + self.pipViewController = nil + } + + private func initialPiPPeer() -> NCPeerConnection? { + // In speaker view the participant shown in Picture in Picture follows the promoted one, + // otherwise it is determined like the initially promoted participant in speaker view + if callViewMode == .speaker, let promotedPeer = peersInCall.first(where: { $0.peerIdentifier == promotedPeerIdentifier }) { + return promotedPeer + } + + return initialPromotedPeer() + } + + private func seedPiPContentSize() { + guard let pipViewController, let pipPeerIdentifier, + let rendererView = videoRenderersDict[pipPeerIdentifier] + else { return } + + // Size the Picture in Picture window based on the last known video size of the + // shown participant before the window is created. A size that is only reported + // while the window is animating in is not applied by AVKit + pipViewController.setVideoContentSize(rendererView.frame.size) + } + + func promotePeerInPiP(_ peer: NCPeerConnection) { + DispatchQueue.main.async { + guard self.isPiPActive, self.pipPeerIdentifier != peer.peerIdentifier else { return } + + self.pipPeerIdentifier = peer.peerIdentifier + self.updatePiPContent() + } + } + + func promoteNextSpeakerInPiPIfNeeded(_ peer: NCPeerConnection) { + DispatchQueue.main.async { + // When the participant shown in Picture in Picture stops speaking, hand the + // promotion over to a participant that is still speaking (same as in speaker view) + guard self.isPiPActive, self.pipPeerIdentifier == peer.peerIdentifier, + let nextSpeaker = self.peersInCall.first(where: { $0.isPeerSpeaking && $0.peerIdentifier != peer.peerIdentifier }) + else { return } + + self.promotePeerInPiP(nextSpeaker) + } + } + + private func updatePiPPeerIfNeeded() { + guard isPiPActive else { return } + + // Make sure the participant shown in Picture in Picture is still in the call + if !peersInCall.contains(where: { $0.peerIdentifier == pipPeerIdentifier }) { + pipPeerIdentifier = initialPiPPeer()?.peerIdentifier + } + + self.updatePiPContent() + } + + private func updatePiPContentIfNeeded(for peer: NCPeerConnection) { + DispatchQueue.main.async { + guard self.isPiPActive, self.pipPeerIdentifier == peer.peerIdentifier else { return } + + self.updatePiPContent() + } + } + + private func reattachPiPRendererIfNeeded(for peer: NCPeerConnection) { + DispatchQueue.main.async { + guard self.isPiPActive, self.pipPeerIdentifier == peer.peerIdentifier else { return } + + // A new remote stream was added for the shown participant, + // so the renderer needs to be attached to the new video track + self.detachPiPRenderer() + self.updatePiPContent() + } + } + + private func updatePiPContent() { + guard isPiPActive, let pipViewController else { return } + + guard let peer = peersInCall.first(where: { $0.peerIdentifier == pipPeerIdentifier }) else { + if peersInCall.isEmpty { + // No other participant in the call, show the conversation name + self.detachPiPRenderer() + pipViewController.showPlaceholder(withDisplayName: self.room.displayName) + } + + // Otherwise the participant was not added to the collection view yet, + // in that case the content is updated when the pending insert is processed + + return + } + + if pipRendererAttachedPeer !== peer { + self.detachPiPRenderer() + self.pipRendererAttachedPeer = peer + + WebRTCCommon.shared.dispatch { + peer.getRemoteStream()?.videoTracks.first?.add(pipViewController.videoRenderView) + } + } + + pipViewController.setVideoDisabled(peer.isRemoteVideoDisabled || !peer.hasRemoteStream()) + + WebRTCCommon.shared.dispatch { + let actor = self.callController?.getActor(fromSessionId: peer.peerId) ?? TalkActor() + + if actor.rawDisplayName.isEmpty, let peerName = peer.peerName, !peerName.isEmpty { + actor.rawDisplayName = peerName + } + + DispatchQueue.main.async { + pipViewController.setAvatar(for: actor, using: self.account) + } + } + } + + private func detachPiPRenderer() { + guard let pipViewController else { return } + + if let attachedPeer = pipRendererAttachedPeer { + WebRTCCommon.shared.dispatch { + attachedPeer.getRemoteStream()?.videoTracks.first?.remove(pipViewController.videoRenderView) + } + } + + self.pipRendererAttachedPeer = nil + pipViewController.videoRenderView.flush() + } + + private func attachPiPLocalRendererIfNeeded() { + guard isPiPActive, let callController, let pipViewController, + callController.isCameraUsableWhileInBackground() + else { return } + + self.pipLocalRendererAttached = true + + pipViewController.localVideoRenderView.isMirrored = callController.isUsingFrontCamera() + callController.attachRenderer(toLocalVideoTrack: pipViewController.localVideoRenderView) + + callController.getVideoEnabledState { isEnabled in + DispatchQueue.main.async { + guard self.pipLocalRendererAttached else { return } + + pipViewController.setLocalVideoHidden(!isEnabled) + } + } + } + + private func detachPiPLocalRenderer() { + guard let pipViewController else { return } + + if pipLocalRendererAttached { + callController?.detachRenderer(fromLocalVideoTrack: pipViewController.localVideoRenderView) + } + + self.pipLocalRendererAttached = false + pipViewController.setLocalVideoHidden(true) + pipViewController.localVideoRenderView.flush() + } + func updateParticipantCell(cell: CallParticipantViewCell, withPeerConnection peerConnection: NCPeerConnection) { var isVideoDisabled = peerConnection.isRemoteVideoDisabled @@ -919,6 +1131,15 @@ class CallViewController: UIViewController, self.userDisabledVideo = true } + DispatchQueue.main.async { + // The local video track might have been recreated while in Picture in Picture + // (e.g. when reconnecting a call), so the renderer needs to be attached to the new track + if self.pipLocalRendererAttached { + self.detachPiPLocalRenderer() + self.attachPiPLocalRendererIfNeeded() + } + } + #if targetEnvironment(simulator) // On the simulator there's no camera controller drawing on localVideoView, // so we render the generated local track with an RTCMTLVideoView instead @@ -970,6 +1191,8 @@ class CallViewController: UIViewController, // This is a new peer, add it self.addPeer(remotePeer) } + + self.reattachPiPRendererIfNeeded(for: remotePeer) } else if remotePeer.roomType == kRoomTypeScreen { self.screenRenderersDict[remotePeer.peerId] = renderView self.screenPeersInCall.append(remotePeer) @@ -1020,6 +1243,8 @@ class CallViewController: UIViewController, self.updatePeer(peer) { cell in cell.videoDisabled = peer.isRemoteVideoDisabled } + + self.updatePiPContentIfNeeded(for: peer) } case "speaking", "stoppedSpeaking": if peersInCall.count > 1 { @@ -1029,9 +1254,11 @@ class CallViewController: UIViewController, // Add to speakers and sort participants if needed if message == "speaking" { self.promotePeerInSpeakerView(peer) + self.promotePeerInPiP(peer) self.addSpeakerAndPromoteIfNeeded(peer) } else { self.promoteNextSpeakerInSpeakerViewIfNeeded(peer) + self.promoteNextSpeakerInPiPIfNeeded(peer) } } case "raiseHand": @@ -1048,6 +1275,8 @@ class CallViewController: UIViewController, cell.displayName = nick } + self.updatePiPContentIfNeeded(for: peer) + if peer.peerId == self.presentedScreenPeerId { DispatchQueue.main.async { self.screenshareLabel.text = nick @@ -1970,6 +2199,12 @@ class CallViewController: UIViewController, self.setLocalVideoViewWrapperHidden(!enabled) self.setVideoDisableButtonActive(enabled) + + DispatchQueue.main.async { + if self.pipLocalRendererAttached { + self.pipViewController?.setLocalVideoHidden(!enabled) + } + } } @IBAction func switchCameraButtonPressed(_ sender: Any) { @@ -2064,6 +2299,8 @@ class CallViewController: UIViewController, self.moreMenuButton?.contextMenuInteraction?.dismissMenu() self.hangUpButton?.contextMenuInteraction?.dismissMenu() + self.stopPictureInPicture() + self.delegate?.callViewControllerWantsToBeDismissed(self) callController?.stopCapturing() @@ -2384,6 +2621,10 @@ class CallViewController: UIViewController, func finishCall() { callController = nil + DispatchQueue.main.async { + self.stopPictureInPicture() + } + if videoCallUpgrade { videoCallUpgrade = false self.delegate?.callViewControllerWantsVideoCallUpgrade(self) @@ -2650,6 +2891,7 @@ class CallViewController: UIViewController, // Don't delay adding the first peer self.peersInCall.append(peer) self.updateSnapshot() + self.updatePiPPeerIfNeeded() } else { // Delay updating the collection view a bit to allow batch updating self.pendingPeerInserts.append(peer) @@ -2725,6 +2967,8 @@ class CallViewController: UIViewController, promotedPeerIdentifier = initialPromotedPeer()?.peerIdentifier } + self.updatePiPPeerIfNeeded() + // Sort peers in call self.sortPeersInCall() @@ -2772,3 +3016,45 @@ class CallViewController: UIViewController, showRoomInfo() } } + +// MARK: - AVPictureInPictureControllerDelegate + +extension CallViewController: AVPictureInPictureControllerDelegate { + + func pictureInPictureControllerWillStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { + self.isPiPActive = true + self.pipPeerIdentifier = self.initialPiPPeer()?.peerIdentifier + self.seedPiPContentSize() + self.updatePiPContent() + self.attachPiPLocalRendererIfNeeded() + } + + func pictureInPictureControllerDidStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { + guard let pipViewController else { return } + + // A video size that was reported while the window was still animating in is + // ignored by AVKit, so apply the current size again now that the window is shown + pipViewController.reassertVideoContentSize() + } + + func pictureInPictureControllerDidStopPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { + self.isPiPActive = false + self.pipPeerIdentifier = nil + self.detachPiPRenderer() + self.detachPiPLocalRenderer() + } + + func pictureInPictureController(_ pictureInPictureController: AVPictureInPictureController, failedToStartPictureInPictureWithError error: Error) { + print("Failed to start Picture in Picture: \(error)") + + self.isPiPActive = false + self.pipPeerIdentifier = nil + self.detachPiPRenderer() + self.detachPiPLocalRenderer() + } + + func pictureInPictureController(_ pictureInPictureController: AVPictureInPictureController, restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completionHandler: @escaping (Bool) -> Void) { + // The call view is still presented fullscreen, nothing to restore + completionHandler(true) + } +} diff --git a/NextcloudTalk/Calls/NCCallController.swift b/NextcloudTalk/Calls/NCCallController.swift index 9c8343bc9..a229fcc9d 100644 --- a/NextcloudTalk/Calls/NCCallController.swift +++ b/NextcloudTalk/Calls/NCCallController.swift @@ -573,6 +573,31 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling } } } + + public func isUsingFrontCamera() -> Bool { + return self.cameraController?.isUsingFrontCamera() ?? false + } + + public func isCameraUsableWhileInBackground() -> Bool { + #if targetEnvironment(simulator) + // The generated test pattern keeps running while the app is in the background + return self.simulatorVideoCapturer != nil + #else + return self.cameraController?.isMultitaskingCameraAccessEnabled() ?? false + #endif + } + + public func attachRenderer(toLocalVideoTrack renderer: RTCVideoRenderer) { + WebRTCCommon.shared.dispatch { + self.localVideoTrack?.add(renderer) + } + } + + public func detachRenderer(fromLocalVideoTrack renderer: RTCVideoRenderer) { + WebRTCCommon.shared.dispatch { + self.localVideoTrack?.remove(renderer) + } + } public func enableVideo(_ enable: Bool) { WebRTCCommon.shared.dispatch { diff --git a/NextcloudTalk/Calls/NCCameraController.swift b/NextcloudTalk/Calls/NCCameraController.swift index 070e25947..089338c14 100644 --- a/NextcloudTalk/Calls/NCCameraController.swift +++ b/NextcloudTalk/Calls/NCCameraController.swift @@ -206,6 +206,12 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate self.session?.sessionPreset = .inputPriority + // Keep the camera running while the app is in the background during + // Picture in Picture on devices that support multitasking camera access + if self.session?.isMultitaskingCameraAccessSupported ?? false { + self.session?.isMultitaskingCameraAccessEnabled = true + } + if let input = self.usingFrontCamera ? self.getFrontCameraInput() : self.getBackCameraInput() { self.session?.addInput(input) } @@ -224,6 +230,14 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate self.session?.stopRunning() } + public func isMultitaskingCameraAccessEnabled() -> Bool { + return session?.isMultitaskingCameraAccessEnabled ?? false + } + + public func isUsingFrontCamera() -> Bool { + return usingFrontCamera + } + // MARK: - Public switches public func enableBackgroundBlur(enable: Bool) { From b2164d2c8ac3efaf3879007e8930a72f3155e10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Thu, 16 Jul 2026 19:04:25 +0200 Subject: [PATCH 3/8] fix: Handle camera interruption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 Signed-off-by: Marcel Müller --- NextcloudTalk/Calls/NCCallController.swift | 25 +++++++++++ NextcloudTalk/Calls/NCCameraController.swift | 46 ++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/NextcloudTalk/Calls/NCCallController.swift b/NextcloudTalk/Calls/NCCallController.swift index a229fcc9d..fc9bf773d 100644 --- a/NextcloudTalk/Calls/NCCallController.swift +++ b/NextcloudTalk/Calls/NCCallController.swift @@ -116,6 +116,7 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling private var localVideoTrack: RTCVideoTrack? private var localScreenTrack: RTCVideoTrack? private var localVideoCaptureController: ARDCaptureController? + private var videoDisabledDueToInterruption = false private let screensharingController = NCScreensharingController() @@ -1496,6 +1497,30 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling self.delegate?.callControllerDidDrawFirstLocalFrame(self) } + func cameraSessionWasInterrupted() { + WebRTCCommon.shared.dispatch { + // When the capture session gets interrupted (e.g. while the Picture in Picture + // window is stashed to the side of the screen), let the other participants + // know that there's no video, so they show the avatar instead of a frozen frame + guard self.isVideoEnabled() else { return } + + self.videoDisabledDueToInterruption = true + self.localVideoTrack?.isEnabled = false + self.sendMessageToAll(ofType: "videoOff", withPayload: nil) + } + } + + func cameraSessionInterruptionEnded() { + WebRTCCommon.shared.dispatch { + // Only enable the video again if it was disabled because of the interruption + guard self.videoDisabledDueToInterruption else { return } + + self.videoDisabledDueToInterruption = false + self.localVideoTrack?.isEnabled = true + self.sendMessageToAll(ofType: "videoOn", withPayload: nil) + } + } + // MARK: - NCPeerConnection delegate // Delegates from NCPeerConnection are already dispatched to the webrtc worker queue diff --git a/NextcloudTalk/Calls/NCCameraController.swift b/NextcloudTalk/Calls/NCCameraController.swift index 089338c14..b04dc6003 100644 --- a/NextcloudTalk/Calls/NCCameraController.swift +++ b/NextcloudTalk/Calls/NCCameraController.swift @@ -12,6 +12,8 @@ import MetalKit @objc protocol NCCameraControllerDelegate { @objc func didDrawFirstFrameOnLocalView() + @objc func cameraSessionWasInterrupted() + @objc func cameraSessionInterruptionEnded() } @objcMembers @@ -71,6 +73,9 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate initAVCaptureSession() NotificationCenter.default.addObserver(self, selector: #selector(deviceOrientationDidChangeNotification), name: UIDevice.orientationDidChangeNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(sessionWasInterrupted(notification:)), name: AVCaptureSession.wasInterruptedNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(sessionInterruptionEnded(notification:)), name: AVCaptureSession.interruptionEndedNotification, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(sessionRuntimeError(notification:)), name: AVCaptureSession.runtimeErrorNotification, object: nil) self.updateVideoRotationBasedOnDeviceOrientation() } @@ -403,6 +408,47 @@ class NCCameraController: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate // MARK: - Notifications + func sessionWasInterrupted(notification: Notification) { + guard let session = notification.object as? AVCaptureSession, session === self.session else { return } + + // The system interrupts the capture session when the camera is not available + // anymore, e.g. when the Picture in Picture window is stashed to the side of + // the screen while using the camera in the background + if let reason = notification.userInfo?[AVCaptureSessionInterruptionReasonKey] as? Int { + print("Capture session was interrupted, reason \(reason)") + } + + delegate?.cameraSessionWasInterrupted() + } + + func sessionInterruptionEnded(notification: Notification) { + guard let session = notification.object as? AVCaptureSession, session === self.session else { return } + + print("Capture session interruption ended") + + // The session should resume on its own when the interruption ended, + // but make sure it is really running again + DispatchQueue.global(qos: .userInitiated).async { [weak self] in + guard let session = self?.session, !session.isRunning else { return } + + session.startRunning() + } + + delegate?.cameraSessionInterruptionEnded() + } + + func sessionRuntimeError(notification: Notification) { + guard let session = notification.object as? AVCaptureSession, session === self.session else { return } + + let error = notification.userInfo?[AVCaptureSessionErrorKey] as? NSError + print("Capture session runtime error: \(String(describing: error))") + + // Try to restart the session after a runtime error + DispatchQueue.global(qos: .userInitiated).async { [weak self] in + self?.session?.startRunning() + } + } + func deviceOrientationDidChangeNotification() { let currentOrientation = UIDevice.current.orientation From 02695fda033c141cfa965c4574b0751c3a9b3286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Thu, 16 Jul 2026 19:34:53 +0200 Subject: [PATCH 4/8] fix: Position and color when no participant is in call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 Signed-off-by: Marcel Müller --- NextcloudTalk/Calls/CallPiPViewController.swift | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/NextcloudTalk/Calls/CallPiPViewController.swift b/NextcloudTalk/Calls/CallPiPViewController.swift index 4b2aca197..9d6c10470 100644 --- a/NextcloudTalk/Calls/CallPiPViewController.swift +++ b/NextcloudTalk/Calls/CallPiPViewController.swift @@ -278,6 +278,12 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { localVideoRenderView.layer.cornerRadius = 8 localVideoRenderView.layer.masksToBounds = true + // Match the waiting screen of the call view, where the theme color is darkened + // by a black overlay with 0.6 alpha (see AvatarBackgroundImageView) + var red: CGFloat = 0, green: CGFloat = 0, blue: CGFloat = 0 + NCAppBranding.themeColor().getRed(&red, green: &green, blue: &blue, alpha: nil) + placeholderView.backgroundColor = UIColor(red: red * 0.4, green: green * 0.4, blue: blue * 0.4, alpha: 1) + avatarImageView.layer.cornerRadius = avatarSize / 2 avatarImageView.layer.masksToBounds = true @@ -323,7 +329,7 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { placeholderView.trailingAnchor.constraint(equalTo: view.trailingAnchor), avatarImageView.centerXAnchor.constraint(equalTo: placeholderView.centerXAnchor), - avatarImageView.centerYAnchor.constraint(equalTo: placeholderView.centerYAnchor, constant: -12), + avatarImageView.centerYAnchor.constraint(equalTo: placeholderView.centerYAnchor, constant: -24), avatarImageView.widthAnchor.constraint(equalToConstant: avatarSize), avatarImageView.heightAnchor.constraint(equalToConstant: avatarSize), From df2e4ae155ff138aed6519ba429f98ef2b8251c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Thu, 16 Jul 2026 19:37:39 +0200 Subject: [PATCH 5/8] fix: Race condition when quickly changing apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcel Müller --- NextcloudTalk/Calls/CallViewController.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/NextcloudTalk/Calls/CallViewController.swift b/NextcloudTalk/Calls/CallViewController.swift index 840b937d2..7d40e3436 100644 --- a/NextcloudTalk/Calls/CallViewController.swift +++ b/NextcloudTalk/Calls/CallViewController.swift @@ -3030,6 +3030,14 @@ extension CallViewController: AVPictureInPictureControllerDelegate { } func pictureInPictureControllerDidStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { + // When quickly switching back to the app while Picture in Picture is still + // animating in, the stop in appDidBecomeActive is missed, because the + // controller was not marked as active there yet. Stop it here instead + if UIApplication.shared.applicationState == .active { + pictureInPictureController.stopPictureInPicture() + return + } + guard let pipViewController else { return } // A video size that was reported while the window was still animating in is From 7fa34685f847bce1a43387eb763e3e77f9947d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Fri, 17 Jul 2026 09:42:36 +0200 Subject: [PATCH 6/8] fix: Avatar size and display MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcel Müller --- .../Calls/CallPiPViewController.swift | 26 ++++++++++++++----- NextcloudTalk/Calls/CallViewController.swift | 4 +-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/NextcloudTalk/Calls/CallPiPViewController.swift b/NextcloudTalk/Calls/CallPiPViewController.swift index 9d6c10470..3c75fb13b 100644 --- a/NextcloudTalk/Calls/CallPiPViewController.swift +++ b/NextcloudTalk/Calls/CallPiPViewController.swift @@ -284,7 +284,6 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { NCAppBranding.themeColor().getRed(&red, green: &green, blue: &blue, alpha: nil) placeholderView.backgroundColor = UIColor(red: red * 0.4, green: green * 0.4, blue: blue * 0.4, alpha: 1) - avatarImageView.layer.cornerRadius = avatarSize / 2 avatarImageView.layer.masksToBounds = true displayNameLabel.textColor = .white @@ -312,6 +311,11 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { self.localVideoLandscapeSizeConstraint = localVideoLandscapeSizeConstraint self.localVideoPortraitSizeConstraint = localVideoPortraitSizeConstraint + // The avatar has a preferred fixed size, but shrinks when the window + // is too small to fit it (e.g. when it was resized by the user) + let avatarPreferredSizeConstraint = avatarImageView.heightAnchor.constraint(equalToConstant: avatarSize) + avatarPreferredSizeConstraint.priority = .defaultHigh + NSLayoutConstraint.activate([ localVideoAspectConstraint, localVideoLandscapeSizeConstraint, @@ -330,8 +334,10 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { avatarImageView.centerXAnchor.constraint(equalTo: placeholderView.centerXAnchor), avatarImageView.centerYAnchor.constraint(equalTo: placeholderView.centerYAnchor, constant: -24), - avatarImageView.widthAnchor.constraint(equalToConstant: avatarSize), - avatarImageView.heightAnchor.constraint(equalToConstant: avatarSize), + avatarPreferredSizeConstraint, + avatarImageView.widthAnchor.constraint(equalTo: avatarImageView.heightAnchor), + avatarImageView.heightAnchor.constraint(lessThanOrEqualTo: placeholderView.heightAnchor, multiplier: 0.4), + avatarImageView.widthAnchor.constraint(lessThanOrEqualTo: placeholderView.widthAnchor, multiplier: 0.4), displayNameLabel.topAnchor.constraint(equalTo: avatarImageView.bottomAnchor, constant: 8), displayNameLabel.leadingAnchor.constraint(greaterThanOrEqualTo: placeholderView.leadingAnchor, constant: 8), @@ -342,7 +348,14 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { override func viewDidLayoutSubviews() { super.viewDidLayoutSubviews() + self.updateLocalVideoSizeConstraint() + + // Keep the avatar round, also when it is scaled down in a small window. + // At this point the nested views did not adopt their new size yet, + // so force a layout of the placeholder before reading the avatar bounds + placeholderView.layoutIfNeeded() + avatarImageView.layer.cornerRadius = avatarImageView.bounds.height / 2 } private func updateLocalVideoSizeConstraint() { @@ -426,9 +439,10 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { displayNameLabel.text = actor.displayName } - public func showPlaceholder(withDisplayName displayName: String?) { - avatarImageView.isHidden = true - displayNameLabel.text = displayName + public func showPlaceholder(for room: NCRoom) { + avatarImageView.isHidden = false + avatarImageView.setAvatar(for: room) + displayNameLabel.text = room.displayName self.setVideoDisabled(true) } diff --git a/NextcloudTalk/Calls/CallViewController.swift b/NextcloudTalk/Calls/CallViewController.swift index 7d40e3436..8e532c909 100644 --- a/NextcloudTalk/Calls/CallViewController.swift +++ b/NextcloudTalk/Calls/CallViewController.swift @@ -912,9 +912,9 @@ class CallViewController: UIViewController, guard let peer = peersInCall.first(where: { $0.peerIdentifier == pipPeerIdentifier }) else { if peersInCall.isEmpty { - // No other participant in the call, show the conversation name + // No other participant in the call, show the conversation avatar and name self.detachPiPRenderer() - pipViewController.showPlaceholder(withDisplayName: self.room.displayName) + pipViewController.showPlaceholder(for: self.room) } // Otherwise the participant was not added to the collection view yet, From de29a81b995b66aff5ced4ba5d4a423d9fcc402c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Fri, 17 Jul 2026 11:24:29 +0200 Subject: [PATCH 7/8] fix: PiP size on creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcel Müller --- .../Calls/CallPiPViewController.swift | 36 +++++++++------ NextcloudTalk/Calls/CallViewController.swift | 44 +++++++++++-------- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/NextcloudTalk/Calls/CallPiPViewController.swift b/NextcloudTalk/Calls/CallPiPViewController.swift index 3c75fb13b..96ff43b89 100644 --- a/NextcloudTalk/Calls/CallPiPViewController.swift +++ b/NextcloudTalk/Calls/CallPiPViewController.swift @@ -41,9 +41,7 @@ class SampleBufferVideoRenderView: UIView, RTCVideoRenderer { private let displayView = DisplayLayerView() private var videoRotation = RTCVideoRotation._0 - - // The (rotated) size of the currently rendered video - public private(set) var videoSize: CGSize = .zero + private var videoSize: CGSize = .zero override init(frame: CGRect) { super.init(frame: frame) @@ -232,10 +230,17 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { private let avatarSize = 80.0 - init() { + public static let defaultContentSize = CGSize(width: 1280, height: 720) + + // The preferred content size must have its final value when the + // AVPictureInPictureController is created: AVKit determines the scale of the window + // from it well before the window is shown and ignores changes until Picture in + // Picture finished starting, so a change in between results in a wrongly scaled + // window on the very first start. Once started, size changes are applied fine + init(initialContentSize: CGSize = CallPiPViewController.defaultContentSize) { super.init(nibName: nil, bundle: nil) - self.preferredContentSize = CGSize(width: 1280, height: 720) + self.preferredContentSize = initialContentSize // Hide here instead of in viewDidLoad: when Picture in Picture starts for the first // time, the own video might be unhidden by the delegate before the view is loaded, @@ -396,7 +401,7 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { // The size is normalized to a fixed dimension, so the window size only depends on the // aspect ratio of the video and not on the resolution of the stream, which usually // starts low and adapts shortly after the stream was established - private static func normalizedContentSize(for videoSize: CGSize) -> CGSize? { + public static func normalizedContentSize(for videoSize: CGSize) -> CGSize? { guard videoSize.width > 0, videoSize.height > 0 else { return nil } let scale = 1280 / max(videoSize.width, videoSize.height) @@ -407,7 +412,7 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { // Sets the preferred content size of the Picture in Picture window from a video size. // Note that the window resize is performed by AVKit without an animation, there is // no API to influence that - public func setVideoContentSize(_ size: CGSize) { + private func setVideoContentSize(_ size: CGSize) { guard let normalizedSize = Self.normalizedContentSize(for: size), normalizedSize != self.preferredContentSize else { return } @@ -415,13 +420,11 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { self.preferredContentSize = normalizedSize } - // Re-assert the size of the currently rendered video: AVKit ignores changes to - // preferredContentSize while the Picture in Picture window is still animating in, - // so the setter needs to be called again even though the value did not change - public func reassertVideoContentSize() { - guard let normalizedSize = Self.normalizedContentSize(for: videoRenderView.videoSize) else { return } - - self.preferredContentSize = normalizedSize + // Assigning an unchanged value on purpose: AVKit ignores size changes while the + // window is animating in, but reacts to the setter call once it is shown + public func reassertContentSize() { + let contentSize = self.preferredContentSize + self.preferredContentSize = contentSize } public func setVideoDisabled(_ disabled: Bool) { @@ -444,6 +447,11 @@ class CallPiPViewController: AVPictureInPictureVideoCallViewController { avatarImageView.setAvatar(for: room) displayNameLabel.text = room.displayName + // Without a video there's no size to follow, use the default window size + if self.preferredContentSize != Self.defaultContentSize { + self.preferredContentSize = Self.defaultContentSize + } + self.setVideoDisabled(true) } } diff --git a/NextcloudTalk/Calls/CallViewController.swift b/NextcloudTalk/Calls/CallViewController.swift index 8e532c909..368f42787 100644 --- a/NextcloudTalk/Calls/CallViewController.swift +++ b/NextcloudTalk/Calls/CallViewController.swift @@ -806,7 +806,17 @@ class CallViewController: UIViewController, func setupPictureInPicture() { guard !isAudioOnly, AVPictureInPictureController.isPictureInPictureSupported() else { return } - let pipViewController = CallPiPViewController() + self.createPictureInPicture(withInitialContentSize: CallPiPViewController.defaultContentSize) + } + + private func createPictureInPicture(withInitialContentSize contentSize: CGSize) { + // The initial window scale is determined from the preferred content size the + // content view controller was created with (see CallPiPViewController.init), + // so both the view controller and the controller need to be recreated when the + // expected video size changes before Picture in Picture is started + self.pipController?.contentSource = nil + + let pipViewController = CallPiPViewController(initialContentSize: contentSize) let contentSource = AVPictureInPictureController.ContentSource(activeVideoCallSourceView: self.collectionView, contentViewController: pipViewController) let pipController = AVPictureInPictureController(contentSource: contentSource) @@ -845,17 +855,6 @@ class CallViewController: UIViewController, return initialPromotedPeer() } - private func seedPiPContentSize() { - guard let pipViewController, let pipPeerIdentifier, - let rendererView = videoRenderersDict[pipPeerIdentifier] - else { return } - - // Size the Picture in Picture window based on the last known video size of the - // shown participant before the window is created. A size that is only reported - // while the window is animating in is not applied by AVKit - pipViewController.setVideoContentSize(rendererView.frame.size) - } - func promotePeerInPiP(_ peer: NCPeerConnection) { DispatchQueue.main.async { guard self.isPiPActive, self.pipPeerIdentifier != peer.peerIdentifier else { return } @@ -2811,6 +2810,18 @@ class CallViewController: UIViewController, participantCell.setRemoteVideoSize(size) } + + // Recreate Picture in Picture when the video size of the expected initial + // participant changes while it is not active, so the window has the correct + // scale and aspect on the very first start (see createPictureInPicture) + if !self.isPiPActive, UIApplication.shared.applicationState == .active, + let pipViewController = self.pipViewController, + self.initialPiPPeer()?.peerIdentifier == peerIdentifier, + let expectedContentSize = CallPiPViewController.normalizedContentSize(for: size), + expectedContentSize != pipViewController.preferredContentSize { + + self.createPictureInPicture(withInitialContentSize: expectedContentSize) + } } for (_, rendererView) in self.screenRenderersDict.filter({ $0.value == mtlVideoView }) { @@ -3024,7 +3035,6 @@ extension CallViewController: AVPictureInPictureControllerDelegate { func pictureInPictureControllerWillStartPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { self.isPiPActive = true self.pipPeerIdentifier = self.initialPiPPeer()?.peerIdentifier - self.seedPiPContentSize() self.updatePiPContent() self.attachPiPLocalRendererIfNeeded() } @@ -3038,11 +3048,9 @@ extension CallViewController: AVPictureInPictureControllerDelegate { return } - guard let pipViewController else { return } - - // A video size that was reported while the window was still animating in is - // ignored by AVKit, so apply the current size again now that the window is shown - pipViewController.reassertVideoContentSize() + // A size change that happened while the window was animating in was ignored + // by AVKit, so assert the current size again now that the window is shown + self.pipViewController?.reassertContentSize() } func pictureInPictureControllerDidStopPictureInPicture(_ pictureInPictureController: AVPictureInPictureController) { From 481ec23a27aadc1128ab72af542caa1823f3d218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Fri, 17 Jul 2026 11:29:10 +0200 Subject: [PATCH 8/8] fix: Recreate PiP controller to fix size issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: ClaudeCode:claude-fable-5 Signed-off-by: Marcel Müller --- NextcloudTalk/Calls/CallViewController.swift | 31 +++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/NextcloudTalk/Calls/CallViewController.swift b/NextcloudTalk/Calls/CallViewController.swift index 368f42787..c2e61cd05 100644 --- a/NextcloudTalk/Calls/CallViewController.swift +++ b/NextcloudTalk/Calls/CallViewController.swift @@ -828,6 +828,23 @@ class CallViewController: UIViewController, self.pipController = pipController } + // Recreate Picture in Picture when the video size of the expected initial participant + // does not match the current content size, so the window has the correct scale and + // aspect on the very first start (see createPictureInPicture). Called when a video + // size changes, but also when peers are added: a size reported while its peer was + // still a pending insert is skipped here, because initialPiPPeer can't return it yet + private func recreatePictureInPictureIfNeeded() { + guard !isPiPActive, UIApplication.shared.applicationState == .active, + let pipViewController, + let initialPeerIdentifier = initialPiPPeer()?.peerIdentifier, + let rendererView = videoRenderersDict[initialPeerIdentifier], + let expectedContentSize = CallPiPViewController.normalizedContentSize(for: rendererView.frame.size), + expectedContentSize != pipViewController.preferredContentSize + else { return } + + self.createPictureInPicture(withInitialContentSize: expectedContentSize) + } + private func stopPictureInPicture() { guard let pipController else { return } @@ -2811,17 +2828,7 @@ class CallViewController: UIViewController, participantCell.setRemoteVideoSize(size) } - // Recreate Picture in Picture when the video size of the expected initial - // participant changes while it is not active, so the window has the correct - // scale and aspect on the very first start (see createPictureInPicture) - if !self.isPiPActive, UIApplication.shared.applicationState == .active, - let pipViewController = self.pipViewController, - self.initialPiPPeer()?.peerIdentifier == peerIdentifier, - let expectedContentSize = CallPiPViewController.normalizedContentSize(for: size), - expectedContentSize != pipViewController.preferredContentSize { - - self.createPictureInPicture(withInitialContentSize: expectedContentSize) - } + self.recreatePictureInPictureIfNeeded() } for (_, rendererView) in self.screenRenderersDict.filter({ $0.value == mtlVideoView }) { @@ -2903,6 +2910,7 @@ class CallViewController: UIViewController, self.peersInCall.append(peer) self.updateSnapshot() self.updatePiPPeerIfNeeded() + self.recreatePictureInPictureIfNeeded() } else { // Delay updating the collection view a bit to allow batch updating self.pendingPeerInserts.append(peer) @@ -2979,6 +2987,7 @@ class CallViewController: UIViewController, } self.updatePiPPeerIfNeeded() + self.recreatePictureInPictureIfNeeded() // Sort peers in call self.sortPeersInCall()