From 760fe11aeb3cb35903ec7f021e149089d861ed3c Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Fri, 9 Oct 2026 16:00:01 +0800 Subject: [PATCH 01/21] Add Foundation Models 27 prompt attachments --- README.md | 31 ++- Sources/AnyLanguageModel/Attachment.swift | 107 +++++++++++ .../LanguageModelSession.swift | 18 +- .../Models/FoundationLanguageModel.swift | 10 +- .../Models/SystemLanguageModel.swift | 52 ++++- Sources/AnyLanguageModel/Prompt.swift | 104 ++++++++-- .../PromptAttachmentTests.swift | 181 ++++++++++++++++++ 7 files changed, 476 insertions(+), 27 deletions(-) create mode 100644 Sources/AnyLanguageModel/Attachment.swift create mode 100644 Tests/AnyLanguageModelTests/PromptAttachmentTests.swift diff --git a/README.md b/README.md index 5a969f0b..6826b869 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,31 @@ pass the model's multimodal projector with `mmprojPath:`. > Image inputs are an AnyLanguageModel extension. > See [Differences from Foundation Models](#differences-from-foundation-models). +### Prompt attachments + +Image attachments use the Foundation Models 27 prompt construction syntax: + +```swift +let prompt = Prompt { + "Describe this image" + Attachment(imageURL: fileURL) +} +``` + +On Apple platforms, `Attachment(CGImage, orientation:)` retains typed image +content and orientation. Prompt builders preserve the order of text and images, +and the OS 26 compatibility transcript continues to store images using its +existing `Transcript.Segment.image` Codable representation. + +`SystemLanguageModel` and `FoundationLanguageModel` bridge prompt attachments +to Foundation Models on OS 27. Earlier system runtimes reject them explicitly. +The system bridge requires local file URLs; other provider adapters retain URL +images in their existing transcript representation. + +This first compatibility surface covers prompt `CGImage` and `imageURL` +attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel +buffers, and newer data attachments remain outside this change. + ### Tool Calling Tool calling is supported by all providers. @@ -611,8 +636,8 @@ say which API they follow. described in that provider's section under [Providers](#providers). - `respond(to:images:)`, `Transcript.ImageSegment`, and `Transcript.Segment.image`: [image inputs](#image-inputs). - Foundation Models 27 adds prompt attachments, - and AnyLanguageModel 2.0 will change to match them. + Foundation Models 27-style [prompt attachments](#prompt-attachments) are also + available for ordered text and image prompt construction. - `ToolExecutionDelegate`, `ToolExecutionDecision`, and `toolExecutionDelegate`: [observing and controlling tool calls](#tool-calling). - `transcriptErrorHandlingPolicy` and `waitForResponseCompletion()`: @@ -645,7 +670,7 @@ say which API they follow. ### Behavior differences - A session adds the prompt to its transcript before the model responds, - as a single text segment. + preserving its text and image segment order. - A structured response is recorded in the transcript as JSON text, not as a structured segment. - A session created from a transcript restores `instructions` diff --git a/Sources/AnyLanguageModel/Attachment.swift b/Sources/AnyLanguageModel/Attachment.swift new file mode 100644 index 00000000..a0cc88c0 --- /dev/null +++ b/Sources/AnyLanguageModel/Attachment.swift @@ -0,0 +1,107 @@ +import Foundation + +#if canImport(CoreGraphics) && canImport(ImageIO) + import CoreGraphics + import ImageIO +#endif + +/// A typed prompt attachment, following Foundation Models 27's image surface. +/// +/// AnyLanguageModel lowers image attachments into its existing compatibility +/// transcript image segments on deployment targets before OS 27. +public struct Attachment { + let content: Content +} + +/// Image content retained until a prompt is lowered into the compatibility transcript. +public struct ImageAttachmentContent: Sendable, Equatable { + enum Source: Sendable { + case url(URL, orientation: UInt32?) + #if canImport(CoreGraphics) && canImport(ImageIO) + case image(CGImage, orientation: CGImagePropertyOrientation?) + #endif + } + + let source: Source + + public static func == (lhs: Self, rhs: Self) -> Bool { + switch (lhs.source, rhs.source) { + case (.url(let a, let ao), .url(let b, let bo)): + return a == b && ao == bo + #if canImport(CoreGraphics) && canImport(ImageIO) + case (.image(let a, let ao), .image(let b, let bo)): + return a === b && ao == bo + #endif + default: + return false + } + } + + func makeSegment(id: String) throws -> Transcript.ImageSegment { + switch source { + case .url(let url, let orientation): + #if canImport(CoreGraphics) && canImport(ImageIO) + if let orientation { + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return try Self( + source: .image( + image, + orientation: CGImagePropertyOrientation(rawValue: orientation) + ) + ).makeSegment(id: id) + } + #endif + return .init(id: id, url: url) + #if canImport(CoreGraphics) && canImport(ImageIO) + case .image(let image, let orientation): + let bytes = NSMutableData() + guard + let destination = CGImageDestinationCreateWithData( + bytes, + "public.png" as CFString, + 1, + nil + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + var properties: [CFString: Any] = [:] + if let orientation { + properties[kCGImagePropertyOrientation] = orientation.rawValue + } + CGImageDestinationAddImage(destination, image, properties as CFDictionary) + guard CGImageDestinationFinalize(destination) else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return .init(id: id, data: bytes as Data, mimeType: "image/png") + #endif + } + } +} + +extension Attachment where Content == ImageAttachmentContent { + #if canImport(CoreGraphics) && canImport(ImageIO) + public init(_ image: CGImage, orientation: CGImagePropertyOrientation? = nil) { + content = .init(source: .image(image, orientation: orientation)) + } + + public init(imageURL: URL, orientation: CGImagePropertyOrientation? = nil) { + // Preserve the URL itself; its bytes are not read merely by constructing a prompt. + content = .init(source: .url(imageURL, orientation: orientation?.rawValue)) + } + #else + public init(imageURL: URL) { + content = .init(source: .url(imageURL, orientation: nil)) + } + #endif +} + +extension Attachment: PromptRepresentable where Content == ImageAttachmentContent { + public var promptRepresentation: Prompt { + Prompt(components: [.image(id: UUID().uuidString, content: content)]) + } +} diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 4f08efc6..b7775ce3 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -728,7 +728,7 @@ public final class LanguageModelSession: @unchecked Sendable { let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( id: promptID, - segments: [.text(.init(content: prompt.description))], + segments: try prompt.makeTranscriptSegments(), options: options, responseFormat: responseFormat ) @@ -776,9 +776,15 @@ public final class LanguageModelSession: @unchecked Sendable { options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream where Content: Generable { // Add prompt to transcript + let segments: [Transcript.Segment] + do { + segments = try prompt.makeTranscriptSegments() + } catch { + return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], + segments: segments, options: options, responseFormat: type == String.self ? nil : .init(type: type) ) @@ -924,9 +930,15 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { + let segments: [Transcript.Segment] + do { + segments = try prompt.makeTranscriptSegments() + } catch { + return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], + segments: segments, options: options, responseFormat: .init(schema: schema) ) diff --git a/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift index b5bf8599..6c54fcec 100644 --- a/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift @@ -159,7 +159,7 @@ ) async throws -> LanguageModelSession.Response where Content: Generable { return try await fmRespond( makeSession: { try await self.makeSession(for: session, prompt: prompt) }, - fmPrompt: prompt.toFoundationModels(), + fmPrompt: try prompt.toFoundationModels(), fmOptions: options.toFoundationModels(), type: type, schema: schema, @@ -209,9 +209,15 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { + let fmPrompt: FoundationModels.Prompt + do { + fmPrompt = try prompt.toFoundationModels() + } catch { + return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } return fmStreamResponse( makeSession: { try await self.makeSession(for: session, prompt: prompt) }, - fmPrompt: prompt.toFoundationModels(), + fmPrompt: fmPrompt, fmOptions: options.toFoundationModels(), type: type, schema: schema, diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 87763ed6..d66ab692 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -194,7 +194,7 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) async throws -> LanguageModelSession.Response where Content: Generable { - let fmPrompt = prompt.toFoundationModels() + let fmPrompt = try prompt.toFoundationModels() let fmOptions = options.toFoundationModels() return try await fmRespond( @@ -249,7 +249,12 @@ includeSchemaInPrompt: Bool, options: GenerationOptions ) -> sending LanguageModelSession.ResponseStream where Content: Generable { - let fmPrompt = prompt.toFoundationModels() + let fmPrompt: FoundationModels.Prompt + do { + fmPrompt = try prompt.toFoundationModels() + } catch { + return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) }) + } let fmOptions = options.toFoundationModels() return fmStreamResponse( @@ -430,8 +435,42 @@ @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) extension Prompt { - func toFoundationModels() -> FoundationModels.Prompt { - FoundationModels.Prompt(self.description) + func toFoundationModels() throws -> FoundationModels.Prompt { + let prompts: [FoundationModels.Prompt] = try components.map { component in + switch component { + case .text(let text): + return FoundationModels.Prompt(text.content) + case .image(_, let image): + #if compiler(>=6.4) && !os(tvOS) && !os(watchOS) + if #available(macOS 27, iOS 27, visionOS 27, *) { + switch image.source { + case .url(let url, let orientation): + guard url.isFileURL else { + throw PromptBridgeError.nonFileImageURL + } + return FoundationModels.Prompt( + FoundationModels.Attachment( + imageURL: url, + orientation: orientation.flatMap( + CGImagePropertyOrientation.init(rawValue:) + ) + ) + ) + case .image(let value, let orientation): + return FoundationModels.Prompt( + FoundationModels.Attachment(value, orientation: orientation) + ) + } + } + #endif + throw PromptBridgeError.attachmentsUnavailable + } + } + return FoundationModels.Prompt { + for prompt in prompts { + prompt + } + } } } @@ -517,6 +556,11 @@ } } + enum PromptBridgeError: Error { + case attachmentsUnavailable + case nonFileImageURL + } + /// A type-erased wrapper that bridges any `Tool` to `FoundationModels.Tool`. @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) private struct AnyToolWrapper: FoundationModels.Tool { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 7e099134..5d215051 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -1,3 +1,5 @@ +import Foundation + /// A prompt from a person to the model. /// /// Prompts can contain content written by you, an outside source, or input directly from people using @@ -24,7 +26,12 @@ /// own prompt to better steer the model's response. For more information on handling inputs in your /// prompts, see . public struct Prompt: Sendable { - private let content: String + enum Component: Sendable { + case text(Transcript.TextSegment) + case image(id: String, content: ImageAttachmentContent) + } + + let components: [Component] /// Creates an instance with the content you specify. public init(_ representable: some PromptRepresentable) { @@ -34,12 +41,76 @@ public struct Prompt: Sendable { case let string as String: self.init(content: string) default: - self.init(content: representable.promptRepresentation.content) + self = representable.promptRepresentation } } init(content: String) { - self.content = content + components = [.text(.init(content: content))] + } + + init(components: [Component]) { + self.components = components + } + + func makeTranscriptSegments() throws -> [Transcript.Segment] { + try components.map { component in + switch component { + case .text(let text): + return .text(text) + case .image(let id, let content): + return .image(try content.makeSegment(id: id)) + } + } + } + + static func joining(_ prompts: [Prompt], trim: Bool = false) -> Prompt { + var components: [Component] = [] + for prompt in prompts { + for component in prompt.components { + if case .text(let text) = component, let last = components.last, + case .text(let previous) = last + { + components[components.count - 1] = .text( + .init( + id: previous.id, + content: previous.content + "\n" + text.content + ) + ) + } else { + components.append(component) + } + } + } + + if trim { + if let first = components.first, case .text(let text) = first { + components[0] = .text( + .init( + id: text.id, + content: text.content.replacingOccurrences( + of: #"^\s+"#, + with: "", + options: .regularExpression + ) + ) + ) + } + if let last = components.last, case .text(let text) = last { + components[components.count - 1] = .text( + .init( + id: text.id, + content: text.content.replacingOccurrences( + of: #"\s+$"#, + with: "", + options: .regularExpression + ) + ) + ) + } + } + + return Prompt(components: components) } public init(@PromptBuilder _ content: () throws -> Prompt) rethrows { @@ -50,7 +121,16 @@ public struct Prompt: Sendable { // MARK: - CustomStringConvertible extension Prompt: CustomStringConvertible { - public var description: String { content } + public var description: String { + components.map { component in + switch component { + case .text(let text): + return text.content + case .image: + return "" + } + }.joined(separator: "\n") + } } // MARK: - PromptBuilder @@ -59,10 +139,9 @@ extension Prompt: CustomStringConvertible { public struct PromptBuilder { public static func buildBlock(_ components: repeat each P) -> Prompt where repeat each P: PromptRepresentable { - var parts: [String] = [] - repeat parts.append((each components).promptRepresentation.description) - let combinedText = parts.joined(separator: "\n") - return Prompt(content: combinedText.trimmingCharacters(in: .whitespacesAndNewlines)) + var parts: [Prompt] = [] + repeat parts.append((each components).promptRepresentation) + return Prompt.joining(parts, trim: true) } public static func buildExpression

(_ expression: P) -> P where P: PromptRepresentable { @@ -70,10 +149,7 @@ public struct PromptBuilder { } public static func buildArray(_ prompts: [some PromptRepresentable]) -> Prompt { - let combinedText = prompts.map { - $0.promptRepresentation.description - }.joined(separator: "\n") - return Prompt(content: combinedText) + Prompt.joining(prompts.map(\.promptRepresentation)) } public static func buildOptional(_ component: Prompt?) -> Prompt { @@ -120,8 +196,6 @@ extension String: PromptRepresentable { extension Array: PromptRepresentable where Element: PromptRepresentable { /// An instance that represents a prompt. public var promptRepresentation: Prompt { - let combined = self.map { $0.promptRepresentation.description } - .joined(separator: "\n") - return Prompt(content: combined) + Prompt.joining(map(\.promptRepresentation)) } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift new file mode 100644 index 00000000..df184ca5 --- /dev/null +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -0,0 +1,181 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +#if canImport(CoreGraphics) && canImport(ImageIO) + import CoreGraphics + import ImageIO +#endif +#if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + import FoundationModels +#endif + +@Suite("Prompt attachments") +struct PromptAttachmentTests { + private let imageURL = URL(fileURLWithPath: "/tmp/attachment-fixture.png") + + private func prompt() -> AnyLanguageModel.Prompt { + AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(imageURL: imageURL) + "after" + } + } + + @Test func buildersAndRepresentablesPreserveOrderedImages() throws { + let original = prompt() + struct Representable: AnyLanguageModel.PromptRepresentable { + let promptRepresentation: AnyLanguageModel.Prompt + } + let wrapped = AnyLanguageModel.Prompt( + Representable(promptRepresentation: original) + ) + let composed = AnyLanguageModel.Prompt { + if true { + wrapped + } + for item in [original] { + item + } + } + + let expected = try original.makeTranscriptSegments() + #expect(try wrapped.makeTranscriptSegments() == expected) + let combined = try composed.makeTranscriptSegments() + #expect(combined.count == 5) + let sources: [AnyLanguageModel.Transcript.ImageSegment.Source] = combined.compactMap { + if case .image(let value) = $0 { + return value.source + } + return nil + } + let texts: [String] = combined.compactMap { + if case .text(let value) = $0 { + return value.content + } + return nil + } + #expect(sources == [.url(imageURL), .url(imageURL)]) + #expect(texts == ["before", "after\nbefore", "after"]) + #expect(original.description == "before\n\nafter") + #expect( + try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() + == combined + ) + } + + @Test func directSessionPromptsRetainImagesInBothModes() async throws { + for streaming in [false, true] { + let session = AnyLanguageModel.LanguageModelSession(model: MockLanguageModel()) + let input = prompt() + if streaming { + _ = try await session.streamResponse(to: input).collect() + } else { + _ = try await session.respond(to: input) + } + + let first = try #require(session.transcript.first) + guard case .prompt(let recorded) = first else { + Issue.record("Missing prompt") + return + } + #expect(recorded.segments == (try input.makeTranscriptSegments())) + } + } + + #if canImport(CoreGraphics) && canImport(ImageIO) + private func image() throws -> CGImage { + let context = try #require( + CGContext( + data: nil, + width: 2, + height: 3, + bitsPerComponent: 8, + bytesPerRow: 8, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + ) + return try #require(context.makeImage()) + } + + @Test func imageEncodingRetainsPixelsAndOrientation() throws { + let value = try image() + let input = AnyLanguageModel.Prompt { + "image note" + AnyLanguageModel.Attachment(value, orientation: .right) + } + let segments = try input.makeTranscriptSegments() + guard case .image(let segment) = segments.last, + case .data(let bytes, let mime) = segment.source + else { + Issue.record("No encoded image") + return + } + #expect(mime == "image/png") + let source = try #require( + CGImageSourceCreateWithData(bytes as CFData, nil) + ) + let decoded = try #require(CGImageSourceCreateImageAtIndex(source, 0, nil)) + #expect(decoded.width == value.width) + #expect(decoded.height == value.height) + let properties = try #require( + CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + ) + #expect( + (properties[kCGImagePropertyOrientation] as? NSNumber)?.uint32Value + == CGImagePropertyOrientation.right.rawValue + ) + } + #endif + + #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func nativeBridgeRetainsAttachment() async throws { + let input = try AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(try image(), orientation: .right) + "after" + } + let session = FoundationModels.LanguageModelSession(model: BridgeModel()) + _ = try await session.respond(to: input.toFoundationModels()) + guard case .prompt(let recorded) = try #require(session.transcript.first) else { + Issue.record("Missing prompt") + return + } + #expect(recorded.segments.count == 3) + guard case .attachment(let segment) = recorded.segments[1], + case .image(let image) = segment.content + else { + Issue.record("Native bridge lost attachment") + return + } + #expect(image.orientation == .right) + } + + @available(macOS 27, iOS 27, visionOS 27, *) + private struct BridgeModel: FoundationModels.LanguageModel { + typealias Executor = BridgeExecutor + var capabilities: FoundationModels.LanguageModelCapabilities { + .init([.vision]) + } + var executorConfiguration: String { + "bridge-fixture" + } + } + + @available(macOS 27, iOS 27, visionOS 27, *) + private struct BridgeExecutor: FoundationModels.LanguageModelExecutor { + init(configuration: String) {} + + func respond( + to request: FoundationModels.LanguageModelExecutorGenerationRequest, + model: BridgeModel, + streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel + ) async throws { + await channel.send(.response(action: .appendText("OK", tokenCount: 1))) + } + } + #endif +} From 379ee3cfca6dd026159d08b507baaecf6453480b Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Sun, 11 Oct 2026 22:07:15 +0800 Subject: [PATCH 02/21] Fix attachment prompt replay --- .../Models/SystemLanguageModel.swift | 18 ++++++--- .../PromptAttachmentTests.swift | 39 ++++++++++++++++++- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index d66ab692..9e17d200 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -341,11 +341,9 @@ guard let lastEntry = transcript.last, case .prompt(let lastPrompt) = lastEntry else { return transcript } - let lastText = lastPrompt.segments.compactMap { segment -> String? in - if case .text(let textSegment) = segment { return textSegment.content } - return nil - }.joined() - guard lastText == prompt.description else { + guard let promptSegments = try? prompt.makeTranscriptSegments(), + lastPrompt.segments == promptSegments + else { return transcript } return Transcript(entries: transcript.dropLast()) @@ -968,7 +966,15 @@ else { return nil } - self.init(cgImage) + let properties = + CGImageSourceCopyPropertiesAtIndex( + imageSource, + 0, + nil + ) as? [CFString: Any] + let orientation = (properties?[kCGImagePropertyOrientation] as? NSNumber) + .flatMap { CGImagePropertyOrientation(rawValue: $0.uint32Value) } + self.init(cgImage, orientation: orientation) } } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index df184ca5..9d6c6a2a 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -154,6 +154,22 @@ struct PromptAttachmentTests { #expect(image.orientation == .right) } + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func foundationWrapperDoesNotDuplicatePromptsAndReplaysOrientation() async throws { + let model = AnyLanguageModel.FoundationLanguageModel(BridgeModel()) + let session = AnyLanguageModel.LanguageModelSession(model: model) + let input = try AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(try image(), orientation: .right) + "after" + } + let first = try await session.respond(to: input) + #expect(first.content == "prompts:1 images:1 right:1") + + let second = try await session.respond(to: "follow-up") + #expect(second.content == "prompts:2 images:1 right:1") + } + @available(macOS 27, iOS 27, visionOS 27, *) private struct BridgeModel: FoundationModels.LanguageModel { typealias Executor = BridgeExecutor @@ -174,7 +190,28 @@ struct PromptAttachmentTests { model: BridgeModel, streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel ) async throws { - await channel.send(.response(action: .appendText("OK", tokenCount: 1))) + let prompts = request.transcript.compactMap { entry -> FoundationModels.Transcript.Prompt? in + guard case .prompt(let prompt) = entry else { return nil } + return prompt + } + let images = prompts.flatMap { prompt in + prompt.segments.compactMap { + segment -> FoundationModels.Transcript.ImageAttachment? in + guard case .attachment(let attachment) = segment, + case .image(let image) = attachment.content + else { return nil } + return image + } + } + let right = images.count { $0.orientation == .right } + await channel.send( + .response( + action: .appendText( + "prompts:\(prompts.count) images:\(images.count) right:\(right)", + tokenCount: 1 + ) + ) + ) } } #endif From 2d1a32b7352d38c111005fc1537fc32282fe8c3f Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Sun, 11 Oct 2026 22:10:10 +0800 Subject: [PATCH 03/21] Preserve attachments in tool outputs --- .../Models/SystemLanguageModel.swift | 4 +- Sources/AnyLanguageModel/Tool.swift | 8 +- .../DynamicInstructionsTests.swift | 2 +- .../PromptAttachmentTests.swift | 92 ++++++++++++++++++- .../ToolExecutionDelegateTests.swift | 2 +- 5 files changed, 99 insertions(+), 9 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 9e17d200..bb451d50 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -563,7 +563,7 @@ @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *) private struct AnyToolWrapper: FoundationModels.Tool { typealias Arguments = FoundationModels.GeneratedContent - typealias Output = String + typealias Output = FoundationModels.Prompt let name: String let description: String @@ -582,7 +582,7 @@ func call(arguments: FoundationModels.GeneratedContent) async throws -> Output { let output = try await wrappedTool.callFunction(arguments: arguments) - return output.promptRepresentation.description + return try output.promptRepresentation.toFoundationModels() } } diff --git a/Sources/AnyLanguageModel/Tool.swift b/Sources/AnyLanguageModel/Tool.swift index 368a9961..59f5b800 100644 --- a/Sources/AnyLanguageModel/Tool.swift +++ b/Sources/AnyLanguageModel/Tool.swift @@ -110,7 +110,10 @@ extension Tool where Self.Arguments: Generable { // MARK: - Helpers extension Tool { - internal func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] { + /// Compatibility integration SPI for provider adapters outside this module. + /// This is not part of Apple's public Tool contract. + @_spi(Compatibility) + public func makeOutputSegments(from arguments: GeneratedContent) async throws -> [Transcript.Segment] { let parsedArguments = try Arguments(arguments) let output = try await call(arguments: parsedArguments) @@ -124,7 +127,6 @@ extension Tool { return [Transcript.Segment.text(.init(content: stringOutput))] } - let fallback = output.promptRepresentation.description - return [Transcript.Segment.text(.init(content: fallback))] + return try output.promptRepresentation.makeTranscriptSegments() } } diff --git a/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift index af43ad37..baba7aa1 100644 --- a/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift +++ b/Tests/AnyLanguageModelTests/DynamicInstructionsTests.swift @@ -1,7 +1,7 @@ import Foundation import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel @Suite("Dynamic instructions") struct DynamicInstructionsTests { diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 9d6c6a2a..ea85a885 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -1,7 +1,7 @@ import Foundation import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel #if canImport(CoreGraphics) && canImport(ImageIO) import CoreGraphics @@ -84,6 +84,29 @@ struct PromptAttachmentTests { } } + @Test func ordinaryToolReturnsOrderedImageSegments() async throws { + let segments = try await ImageTool(output: prompt()).makeOutputSegments( + from: AnyLanguageModel.GeneratedContent("read") + ) + #expect(segments.count == 3) + guard case .text(let before) = segments[0], + case .image(let image) = segments[1], + case .text(let after) = segments[2] + else { + Issue.record("Tool output lost its ordered image") + return + } + #expect(before.content == "before") + #expect(after.content == "after") + #expect(image.source == .url(imageURL)) + + let transcript = AnyLanguageModel.Transcript(entries: [ + .toolOutput(.init(id: "image-call", toolName: "image", segments: segments)) + ]) + let encoded = try JSONEncoder().encode(transcript) + #expect(try JSONDecoder().decode(AnyLanguageModel.Transcript.self, from: encoded) == transcript) + } + #if canImport(CoreGraphics) && canImport(ImageIO) private func image() throws -> CGImage { let context = try #require( @@ -170,11 +193,41 @@ struct PromptAttachmentTests { #expect(second.content == "prompts:2 images:1 right:1") } + @available(macOS 27, iOS 27, visionOS 27, *) + @Test func nativeToolWrapperRetainsOrdinaryImageOutput() async throws { + let output = try AnyLanguageModel.Prompt { + "tool note" + AnyLanguageModel.Attachment(try image(), orientation: .right) + } + let tools: [any AnyLanguageModel.Tool] = [ImageTool(output: output)] + let session = FoundationModels.LanguageModelSession( + model: BridgeModel(callsTools: true), + tools: tools.toFoundationModels() + ) + _ = try await session.respond(to: "read image") + let recorded = try #require( + session.transcript.compactMap { entry -> FoundationModels.Transcript.ToolOutput? in + guard case .toolOutput(let output) = entry else { return nil } + return output + }.first + ) + #expect(recorded.id == "image-call") + #expect(recorded.segments.count == 2) + guard case .attachment(let segment) = recorded.segments[1], + case .image(let image) = segment.content + else { + Issue.record("Native Tool wrapper flattened image output") + return + } + #expect(image.orientation == .right) + } + @available(macOS 27, iOS 27, visionOS 27, *) private struct BridgeModel: FoundationModels.LanguageModel { typealias Executor = BridgeExecutor + var callsTools = false var capabilities: FoundationModels.LanguageModelCapabilities { - .init([.vision]) + .init([.vision, .toolCalling]) } var executorConfiguration: String { "bridge-fixture" @@ -190,6 +243,27 @@ struct PromptAttachmentTests { model: BridgeModel, streamingInto channel: FoundationModels.LanguageModelExecutorGenerationChannel ) async throws { + if model.callsTools, + !request.transcript.contains(where: { + if case .toolOutput = $0 { return true } + return false + }) + { + await channel.send( + .toolCalls( + action: .toolCall( + id: "image-call", + name: "image", + action: .appendArguments(#""read""#, tokenCount: 1) + ) + ) + ) + return + } + if model.callsTools { + await channel.send(.response(action: .appendText("OK", tokenCount: 1))) + return + } let prompts = request.transcript.compactMap { entry -> FoundationModels.Transcript.Prompt? in guard case .prompt(let prompt) = entry else { return nil } return prompt @@ -216,3 +290,17 @@ struct PromptAttachmentTests { } #endif } + +private struct ImageTool: AnyLanguageModel.Tool { + let name = "image" + let description = "Return an image" + let output: AnyLanguageModel.Prompt + + var parameters: AnyLanguageModel.GenerationSchema { + String.generationSchema + } + + func call(arguments: String) async throws -> AnyLanguageModel.Prompt { + output + } +} diff --git a/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift b/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift index 1b258e32..383cca86 100644 --- a/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift +++ b/Tests/AnyLanguageModelTests/ToolExecutionDelegateTests.swift @@ -1,6 +1,6 @@ import Testing -@testable import AnyLanguageModel +@_spi(Compatibility) @testable import AnyLanguageModel private actor ToolExecutionDelegateSpy: ToolExecutionDelegate { private(set) var generatedToolCalls: [Transcript.ToolCall] = [] From dc79bbf45827015c13754a036f67645e6ecac0ff Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 12 Oct 2026 03:09:26 +0800 Subject: [PATCH 04/21] Document attachments in tool outputs --- README.md | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6826b869..02bf252b 100644 --- a/README.md +++ b/README.md @@ -384,28 +384,31 @@ pass the model's multimodal projector with `mmprojPath:`. ### Prompt attachments -Image attachments use the Foundation Models 27 prompt construction syntax: +Image attachments use the Foundation Models 27 prompt construction syntax, +including ordinary Tool outputs: ```swift -let prompt = Prompt { - "Describe this image" - Attachment(imageURL: fileURL) +func call(arguments: Arguments) async throws -> Prompt { + Prompt { + "Image read from the selected file" + Attachment(imageURL: fileURL) + } } ``` On Apple platforms, `Attachment(CGImage, orientation:)` retains typed image -content and orientation. Prompt builders preserve the order of text and images, -and the OS 26 compatibility transcript continues to store images using its -existing `Transcript.Segment.image` Codable representation. +content and orientation. Prompts, builders, and Tool outputs preserve the order +of text and images, and the OS 26 compatibility transcript continues to store +images using its existing `Transcript.Segment.image` Codable representation. `SystemLanguageModel` and `FoundationLanguageModel` bridge prompt attachments to Foundation Models on OS 27. Earlier system runtimes reject them explicitly. The system bridge requires local file URLs; other provider adapters retain URL images in their existing transcript representation. -This first compatibility surface covers prompt `CGImage` and `imageURL` -attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel -buffers, and newer data attachments remain outside this change. +This compatibility surface covers `CGImage` and `imageURL` attachments in +prompts and ordinary Tool outputs. Instruction attachments, labels, `CIImage`, +pixel buffers, and newer data attachments remain outside this change. ### Tool Calling @@ -638,6 +641,10 @@ say which API they follow. [image inputs](#image-inputs). Foundation Models 27-style [prompt attachments](#prompt-attachments) are also available for ordered text and image prompt construction. +- `Tool.makeOutputSegments(from:)`, exposed only through the `Compatibility` SPI: + an integration extension for back-deployment provider adapters to lower normal + Tool outputs into the compatibility transcript. It is not an Apple Tool API; + OS 27 canonical executors use Foundation Models' own Tool execution instead. - `ToolExecutionDelegate`, `ToolExecutionDecision`, and `toolExecutionDelegate`: [observing and controlling tool calls](#tool-calling). - `transcriptErrorHandlingPolicy` and `waitForResponseCompletion()`: From f96d6decb4be0eca99af2741e8f8d2b2ba9ee524 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 12 Oct 2026 09:04:53 +0800 Subject: [PATCH 05/21] Generate fresh prompt segment identifiers --- .../Models/SystemLanguageModel.swift | 20 ++++++++++- Sources/AnyLanguageModel/Prompt.swift | 6 ++-- .../PromptAttachmentTests.swift | 35 +++++++++++++++---- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index bb451d50..fb176823 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -342,13 +342,31 @@ return transcript } guard let promptSegments = try? prompt.makeTranscriptSegments(), - lastPrompt.segments == promptSegments + lastPrompt.segments.hasSamePayload(as: promptSegments) else { return transcript } return Transcript(entries: transcript.dropLast()) } + private extension Array where Element == Transcript.Segment { + func hasSamePayload(as other: [Transcript.Segment]) -> Bool { + guard count == other.count else { return false } + return zip(self, other).allSatisfy { lhs, rhs in + switch (lhs, rhs) { + case (.text(let lhs), .text(let rhs)): + lhs.content == rhs.content + case (.structure(let lhs), .structure(let rhs)): + lhs.source == rhs.source && lhs.content == rhs.content + case (.image(let lhs), .image(let rhs)): + lhs.source == rhs.source + default: + false + } + } + } + } + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @available(watchOS, unavailable) extension SystemLanguageModel { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 5d215051..3a829982 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -57,9 +57,9 @@ public struct Prompt: Sendable { try components.map { component in switch component { case .text(let text): - return .text(text) - case .image(let id, let content): - return .image(try content.makeSegment(id: id)) + return .text(.init(content: text.content)) + case .image(_, let content): + return .image(try content.makeSegment(id: UUID().uuidString)) } } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index ea85a885..8f91183c 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -13,8 +13,28 @@ import Testing @Suite("Prompt attachments") struct PromptAttachmentTests { + private enum SegmentPayload: Equatable { + case text(String) + case image(AnyLanguageModel.Transcript.ImageSegment.Source) + } + private let imageURL = URL(fileURLWithPath: "/tmp/attachment-fixture.png") + private func payloads( + _ segments: [AnyLanguageModel.Transcript.Segment] + ) -> [SegmentPayload] { + segments.compactMap { segment in + switch segment { + case .text(let text): + .text(text.content) + case .image(let image): + .image(image.source) + case .structure: + nil + } + } + } + private func prompt() -> AnyLanguageModel.Prompt { AnyLanguageModel.Prompt { "before" @@ -41,9 +61,12 @@ struct PromptAttachmentTests { } let expected = try original.makeTranscriptSegments() - #expect(try wrapped.makeTranscriptSegments() == expected) + let wrappedSegments = try wrapped.makeTranscriptSegments() + #expect(payloads(wrappedSegments) == payloads(expected)) + #expect(Set(wrappedSegments.map(\.id)).isDisjoint(with: expected.map(\.id))) let combined = try composed.makeTranscriptSegments() #expect(combined.count == 5) + #expect(Set(combined.map(\.id)).count == combined.count) let sources: [AnyLanguageModel.Transcript.ImageSegment.Source] = combined.compactMap { if case .image(let value) = $0 { return value.source @@ -59,10 +82,10 @@ struct PromptAttachmentTests { #expect(sources == [.url(imageURL), .url(imageURL)]) #expect(texts == ["before", "after\nbefore", "after"]) #expect(original.description == "before\n\nafter") - #expect( - try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() - == combined - ) + let repeated = try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() + #expect(payloads(repeated) == payloads(combined)) + #expect(Set(repeated.map(\.id)).count == repeated.count) + #expect(Set(repeated.map(\.id)).isDisjoint(with: combined.map(\.id))) } @Test func directSessionPromptsRetainImagesInBothModes() async throws { @@ -80,7 +103,7 @@ struct PromptAttachmentTests { Issue.record("Missing prompt") return } - #expect(recorded.segments == (try input.makeTranscriptSegments())) + #expect(payloads(recorded.segments) == payloads(try input.makeTranscriptSegments())) } } From c7aaa57799f619a0a623abf53bd54d00f8306931 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 12 Oct 2026 09:10:14 +0800 Subject: [PATCH 06/21] Preserve Gemini tool output segments --- README.md | 7 +- .../Models/GeminiLanguageModel.swift | 52 ++++++++++---- .../Models/SystemLanguageModel.swift | 11 ++- .../GeminiThoughtSignatureTests.swift | 71 +++++++++++++++++++ .../PromptAttachmentTests.swift | 11 +++ 5 files changed, 134 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6826b869..6c4eb428 100644 --- a/README.md +++ b/README.md @@ -403,9 +403,10 @@ to Foundation Models on OS 27. Earlier system runtimes reject them explicitly. The system bridge requires local file URLs; other provider adapters retain URL images in their existing transcript representation. -This first compatibility surface covers prompt `CGImage` and `imageURL` -attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel -buffers, and newer data attachments remain outside this change. +This compatibility surface covers prompt `CGImage` and `imageURL` attachments, +including attachments returned from ordinary `Tool` output values. Instruction +attachments, labels, `CIImage`, pixel buffers, and newer data attachments remain +outside this change. ### Tool Calling diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 5fc12db9..bdc5da7f 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -769,26 +769,47 @@ private func emptyResponseContent( } } -private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String: JSONValue] { - var result: [String: JSONValue] = [:] +private func toGeminiFunctionResponse( + _ toolOutput: Transcript.ToolOutput +) throws -> GeminiFunctionResponse { + var resultValues: [String] = [] + var parts: [GeminiPart] = [] for segment in toolOutput.segments { switch segment { case .text(let text): - result["result"] = .string(text.content) + resultValues.append(text.content) case .structure(let structured): - // For structured segments, encode the content let data = try JSONEncoder().encode(structured.content) if let jsonString = String(data: data, encoding: .utf8) { - result["result"] = .string(jsonString) + resultValues.append(jsonString) + } + case .image(let image): + switch image.source { + case .data(let data, let mimeType): + parts.append( + .inlineData( + GeminiInlineData( + mimeType: mimeType, + data: data.base64EncodedString() + ) + ) + ) + case .url(let url): + parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString))) } - case .image: - // Ignore images in tool outputs for Gemini conversion - break } } - return result + var response: [String: JSONValue] = [:] + if !resultValues.isEmpty { + response["result"] = .string(resultValues.joined(separator: "\n")) + } + return GeminiFunctionResponse( + name: toolOutput.toolName, + response: response, + parts: parts.isEmpty ? nil : parts + ) } // MARK: - Supporting Types @@ -844,11 +865,13 @@ extension Transcript { ) case .toolOutput(let toolOutput): // Add function response as a user message (Gemini API expects function responses from user role) - let response = try? toJSONValue(toolOutput) - let functionResponse = GeminiFunctionResponse( - name: toolOutput.toolName, - response: response ?? [:] - ) + let functionResponse = + (try? toGeminiFunctionResponse(toolOutput)) + ?? GeminiFunctionResponse( + name: toolOutput.toolName, + response: [:], + parts: nil + ) messages.append( .init( role: .user, @@ -1116,6 +1139,7 @@ private struct GeminiFunctionCall: Codable, Sendable { private struct GeminiFunctionResponse: Codable, Sendable { let name: String let response: [String: JSONValue] + let parts: [GeminiPart]? } private struct GeminiGenerateContentResponse: Codable, Sendable { diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index fb176823..7354e824 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -572,9 +572,18 @@ } } - enum PromptBridgeError: Error { + enum PromptBridgeError: LocalizedError, Sendable, Equatable { case attachmentsUnavailable case nonFileImageURL + + var errorDescription: String? { + switch self { + case .attachmentsUnavailable: + "Prompt attachments require an OS 27 Foundation Models runtime." + case .nonFileImageURL: + "Foundation Models prompt attachments require a local file URL." + } + } } /// A type-erased wrapper that bridges any `Tool` to `FoundationModels.Tool`. diff --git a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift index 5265718f..d1a93925 100644 --- a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift @@ -4,11 +4,27 @@ import Testing @testable import AnyLanguageModel #if canImport(Darwin) && !canImport(AsyncHTTPClient) + import CoreGraphics @Suite("GeminiLanguageModel thought signatures", .serialized) struct GeminiThoughtSignatureTests { private static let signature = "CvsBAdHtim8n5xQK1pVX2H0lPQeXAMPLEsignature==" + @Generable + struct ImageArguments { + var request: String + } + + private struct ImageOutputTool: Tool { + let name = "image" + let description = "Return an image" + let output: Prompt + + func call(arguments: ImageArguments) async throws -> Prompt { + output + } + } + private func makeModel() -> GeminiLanguageModel { GeminiLanguageModel( apiKey: "test-key", @@ -51,6 +67,21 @@ import Testing """ } + private func image() throws -> CGImage { + let context = try #require( + CGContext( + data: nil, + width: 2, + height: 2, + bitsPerComponent: 8, + bytesPerRow: 8, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + ) + return try #require(context.makeImage()) + } + /// Signatures of every `functionCall` part in a request body, in order. /// A part without a signature contributes `nil`. private func functionCallSignatures(in body: Data) throws -> [String?] { @@ -108,6 +139,46 @@ import Testing #expect(signatures.allSatisfy { $0 == Self.signature }) } + @Test("preserves all text and image data in function responses") + func preservesSegmentedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + StubURLProtocol.enqueue(json: textResponse("OK")) + + let output = try Prompt { + "before" + Attachment(try image()) + "after" + } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + _ = try await session.respond(to: "read image") + + let bodies = StubURLProtocol.recordedBodies + try #require(bodies.count == 2) + let json = try #require( + try JSONSerialization.jsonObject(with: bodies[1]) as? [String: Any] + ) + let contents = try #require(json["contents"] as? [[String: Any]]) + let outputContent = try #require(contents.last) + let outputParts = try #require(outputContent["parts"] as? [[String: Any]]) + let functionResponse = try #require( + outputParts.first?["functionResponse"] as? [String: Any] + ) + let response = try #require(functionResponse["response"] as? [String: Any]) + #expect(response["result"] as? String == "before\nafter") + + let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]]) + let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any]) + #expect(inlineData["mime_type"] as? String == "image/png") + #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) != nil) + } + @Test("does not replay the conversation history on later turns") func doesNotReplayHistoryOnLaterTurns() async throws { StubURLProtocol.reset() diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 8f91183c..81a3b3f7 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -177,6 +177,17 @@ struct PromptAttachmentTests { #endif #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + @Test func promptBridgeErrorsHaveLocalizedDescriptions() { + #expect( + AnyLanguageModel.PromptBridgeError.attachmentsUnavailable.errorDescription + == "Prompt attachments require an OS 27 Foundation Models runtime." + ) + #expect( + AnyLanguageModel.PromptBridgeError.nonFileImageURL.errorDescription + == "Foundation Models prompt attachments require a local file URL." + ) + } + @available(macOS 27, iOS 27, visionOS 27, *) @Test func nativeBridgeRetainsAttachment() async throws { let input = try AnyLanguageModel.Prompt { From d70bf125ffb4d95e5d2329af677721a3efd7d60b Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 12 Oct 2026 09:04:53 +0800 Subject: [PATCH 07/21] Generate fresh prompt segment identifiers --- .../Models/SystemLanguageModel.swift | 20 ++++++++++- Sources/AnyLanguageModel/Prompt.swift | 6 ++-- .../PromptAttachmentTests.swift | 35 +++++++++++++++---- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index bb451d50..fb176823 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -342,13 +342,31 @@ return transcript } guard let promptSegments = try? prompt.makeTranscriptSegments(), - lastPrompt.segments == promptSegments + lastPrompt.segments.hasSamePayload(as: promptSegments) else { return transcript } return Transcript(entries: transcript.dropLast()) } + private extension Array where Element == Transcript.Segment { + func hasSamePayload(as other: [Transcript.Segment]) -> Bool { + guard count == other.count else { return false } + return zip(self, other).allSatisfy { lhs, rhs in + switch (lhs, rhs) { + case (.text(let lhs), .text(let rhs)): + lhs.content == rhs.content + case (.structure(let lhs), .structure(let rhs)): + lhs.source == rhs.source && lhs.content == rhs.content + case (.image(let lhs), .image(let rhs)): + lhs.source == rhs.source + default: + false + } + } + } + } + @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @available(watchOS, unavailable) extension SystemLanguageModel { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 5d215051..3a829982 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -57,9 +57,9 @@ public struct Prompt: Sendable { try components.map { component in switch component { case .text(let text): - return .text(text) - case .image(let id, let content): - return .image(try content.makeSegment(id: id)) + return .text(.init(content: text.content)) + case .image(_, let content): + return .image(try content.makeSegment(id: UUID().uuidString)) } } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index ea85a885..8f91183c 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -13,8 +13,28 @@ import Testing @Suite("Prompt attachments") struct PromptAttachmentTests { + private enum SegmentPayload: Equatable { + case text(String) + case image(AnyLanguageModel.Transcript.ImageSegment.Source) + } + private let imageURL = URL(fileURLWithPath: "/tmp/attachment-fixture.png") + private func payloads( + _ segments: [AnyLanguageModel.Transcript.Segment] + ) -> [SegmentPayload] { + segments.compactMap { segment in + switch segment { + case .text(let text): + .text(text.content) + case .image(let image): + .image(image.source) + case .structure: + nil + } + } + } + private func prompt() -> AnyLanguageModel.Prompt { AnyLanguageModel.Prompt { "before" @@ -41,9 +61,12 @@ struct PromptAttachmentTests { } let expected = try original.makeTranscriptSegments() - #expect(try wrapped.makeTranscriptSegments() == expected) + let wrappedSegments = try wrapped.makeTranscriptSegments() + #expect(payloads(wrappedSegments) == payloads(expected)) + #expect(Set(wrappedSegments.map(\.id)).isDisjoint(with: expected.map(\.id))) let combined = try composed.makeTranscriptSegments() #expect(combined.count == 5) + #expect(Set(combined.map(\.id)).count == combined.count) let sources: [AnyLanguageModel.Transcript.ImageSegment.Source] = combined.compactMap { if case .image(let value) = $0 { return value.source @@ -59,10 +82,10 @@ struct PromptAttachmentTests { #expect(sources == [.url(imageURL), .url(imageURL)]) #expect(texts == ["before", "after\nbefore", "after"]) #expect(original.description == "before\n\nafter") - #expect( - try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() - == combined - ) + let repeated = try AnyLanguageModel.Prompt([original, original]).makeTranscriptSegments() + #expect(payloads(repeated) == payloads(combined)) + #expect(Set(repeated.map(\.id)).count == repeated.count) + #expect(Set(repeated.map(\.id)).isDisjoint(with: combined.map(\.id))) } @Test func directSessionPromptsRetainImagesInBothModes() async throws { @@ -80,7 +103,7 @@ struct PromptAttachmentTests { Issue.record("Missing prompt") return } - #expect(recorded.segments == (try input.makeTranscriptSegments())) + #expect(payloads(recorded.segments) == payloads(try input.makeTranscriptSegments())) } } From ddfa8f1f09071acf0a15b847c8f8537c1dd4471d Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 12 Oct 2026 09:10:14 +0800 Subject: [PATCH 08/21] Preserve Gemini tool output segments --- .../Models/GeminiLanguageModel.swift | 52 ++++++++++---- .../Models/SystemLanguageModel.swift | 11 ++- .../GeminiThoughtSignatureTests.swift | 71 +++++++++++++++++++ .../PromptAttachmentTests.swift | 11 +++ 4 files changed, 130 insertions(+), 15 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 5fc12db9..bdc5da7f 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -769,26 +769,47 @@ private func emptyResponseContent( } } -private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String: JSONValue] { - var result: [String: JSONValue] = [:] +private func toGeminiFunctionResponse( + _ toolOutput: Transcript.ToolOutput +) throws -> GeminiFunctionResponse { + var resultValues: [String] = [] + var parts: [GeminiPart] = [] for segment in toolOutput.segments { switch segment { case .text(let text): - result["result"] = .string(text.content) + resultValues.append(text.content) case .structure(let structured): - // For structured segments, encode the content let data = try JSONEncoder().encode(structured.content) if let jsonString = String(data: data, encoding: .utf8) { - result["result"] = .string(jsonString) + resultValues.append(jsonString) + } + case .image(let image): + switch image.source { + case .data(let data, let mimeType): + parts.append( + .inlineData( + GeminiInlineData( + mimeType: mimeType, + data: data.base64EncodedString() + ) + ) + ) + case .url(let url): + parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString))) } - case .image: - // Ignore images in tool outputs for Gemini conversion - break } } - return result + var response: [String: JSONValue] = [:] + if !resultValues.isEmpty { + response["result"] = .string(resultValues.joined(separator: "\n")) + } + return GeminiFunctionResponse( + name: toolOutput.toolName, + response: response, + parts: parts.isEmpty ? nil : parts + ) } // MARK: - Supporting Types @@ -844,11 +865,13 @@ extension Transcript { ) case .toolOutput(let toolOutput): // Add function response as a user message (Gemini API expects function responses from user role) - let response = try? toJSONValue(toolOutput) - let functionResponse = GeminiFunctionResponse( - name: toolOutput.toolName, - response: response ?? [:] - ) + let functionResponse = + (try? toGeminiFunctionResponse(toolOutput)) + ?? GeminiFunctionResponse( + name: toolOutput.toolName, + response: [:], + parts: nil + ) messages.append( .init( role: .user, @@ -1116,6 +1139,7 @@ private struct GeminiFunctionCall: Codable, Sendable { private struct GeminiFunctionResponse: Codable, Sendable { let name: String let response: [String: JSONValue] + let parts: [GeminiPart]? } private struct GeminiGenerateContentResponse: Codable, Sendable { diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index fb176823..7354e824 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -572,9 +572,18 @@ } } - enum PromptBridgeError: Error { + enum PromptBridgeError: LocalizedError, Sendable, Equatable { case attachmentsUnavailable case nonFileImageURL + + var errorDescription: String? { + switch self { + case .attachmentsUnavailable: + "Prompt attachments require an OS 27 Foundation Models runtime." + case .nonFileImageURL: + "Foundation Models prompt attachments require a local file URL." + } + } } /// A type-erased wrapper that bridges any `Tool` to `FoundationModels.Tool`. diff --git a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift index 5265718f..d1a93925 100644 --- a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift @@ -4,11 +4,27 @@ import Testing @testable import AnyLanguageModel #if canImport(Darwin) && !canImport(AsyncHTTPClient) + import CoreGraphics @Suite("GeminiLanguageModel thought signatures", .serialized) struct GeminiThoughtSignatureTests { private static let signature = "CvsBAdHtim8n5xQK1pVX2H0lPQeXAMPLEsignature==" + @Generable + struct ImageArguments { + var request: String + } + + private struct ImageOutputTool: Tool { + let name = "image" + let description = "Return an image" + let output: Prompt + + func call(arguments: ImageArguments) async throws -> Prompt { + output + } + } + private func makeModel() -> GeminiLanguageModel { GeminiLanguageModel( apiKey: "test-key", @@ -51,6 +67,21 @@ import Testing """ } + private func image() throws -> CGImage { + let context = try #require( + CGContext( + data: nil, + width: 2, + height: 2, + bitsPerComponent: 8, + bytesPerRow: 8, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + ) + ) + return try #require(context.makeImage()) + } + /// Signatures of every `functionCall` part in a request body, in order. /// A part without a signature contributes `nil`. private func functionCallSignatures(in body: Data) throws -> [String?] { @@ -108,6 +139,46 @@ import Testing #expect(signatures.allSatisfy { $0 == Self.signature }) } + @Test("preserves all text and image data in function responses") + func preservesSegmentedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + StubURLProtocol.enqueue(json: textResponse("OK")) + + let output = try Prompt { + "before" + Attachment(try image()) + "after" + } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + _ = try await session.respond(to: "read image") + + let bodies = StubURLProtocol.recordedBodies + try #require(bodies.count == 2) + let json = try #require( + try JSONSerialization.jsonObject(with: bodies[1]) as? [String: Any] + ) + let contents = try #require(json["contents"] as? [[String: Any]]) + let outputContent = try #require(contents.last) + let outputParts = try #require(outputContent["parts"] as? [[String: Any]]) + let functionResponse = try #require( + outputParts.first?["functionResponse"] as? [String: Any] + ) + let response = try #require(functionResponse["response"] as? [String: Any]) + #expect(response["result"] as? String == "before\nafter") + + let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]]) + let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any]) + #expect(inlineData["mime_type"] as? String == "image/png") + #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) != nil) + } + @Test("does not replay the conversation history on later turns") func doesNotReplayHistoryOnLaterTurns() async throws { StubURLProtocol.reset() diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 8f91183c..81a3b3f7 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -177,6 +177,17 @@ struct PromptAttachmentTests { #endif #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) + @Test func promptBridgeErrorsHaveLocalizedDescriptions() { + #expect( + AnyLanguageModel.PromptBridgeError.attachmentsUnavailable.errorDescription + == "Prompt attachments require an OS 27 Foundation Models runtime." + ) + #expect( + AnyLanguageModel.PromptBridgeError.nonFileImageURL.errorDescription + == "Foundation Models prompt attachments require a local file URL." + ) + } + @available(macOS 27, iOS 27, visionOS 27, *) @Test func nativeBridgeRetainsAttachment() async throws { let input = try AnyLanguageModel.Prompt { From d9de68af09d1a21e44d5a095423f9c5fa347dac7 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 13 Oct 2026 09:08:28 +0800 Subject: [PATCH 09/21] Avoid relowering recorded prompt attachments --- .../LanguageModelSession.swift | 10 ++-- .../Models/SystemLanguageModel.swift | 25 ++------- Sources/AnyLanguageModel/Prompt.swift | 28 ++++++++-- .../PromptAttachmentTests.swift | 51 +++++++++++++++++++ 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index b7775ce3..552c3fe0 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -728,7 +728,7 @@ public final class LanguageModelSession: @unchecked Sendable { let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( id: promptID, - segments: try prompt.makeTranscriptSegments(), + segments: try prompt.makeTranscriptSegments(occurrenceID: promptID), options: options, responseFormat: responseFormat ) @@ -776,14 +776,16 @@ public final class LanguageModelSession: @unchecked Sendable { options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream where Content: Generable { // Add prompt to transcript + let promptID = UUID().uuidString let segments: [Transcript.Segment] do { - segments = try prompt.makeTranscriptSegments() + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) } catch { return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( + id: promptID, segments: segments, options: options, responseFormat: type == String.self ? nil : .init(type: type) @@ -930,14 +932,16 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { + let promptID = UUID().uuidString let segments: [Transcript.Segment] do { - segments = try prompt.makeTranscriptSegments() + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) } catch { return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( + id: promptID, segments: segments, options: options, responseFormat: .init(schema: schema) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 7354e824..2eb7e785 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -341,32 +341,17 @@ guard let lastEntry = transcript.last, case .prompt(let lastPrompt) = lastEntry else { return transcript } - guard let promptSegments = try? prompt.makeTranscriptSegments(), - lastPrompt.segments.hasSamePayload(as: promptSegments) + guard + prompt.matchesTranscriptOccurrence( + lastPrompt.segments, + occurrenceID: lastPrompt.id + ) else { return transcript } return Transcript(entries: transcript.dropLast()) } - private extension Array where Element == Transcript.Segment { - func hasSamePayload(as other: [Transcript.Segment]) -> Bool { - guard count == other.count else { return false } - return zip(self, other).allSatisfy { lhs, rhs in - switch (lhs, rhs) { - case (.text(let lhs), .text(let rhs)): - lhs.content == rhs.content - case (.structure(let lhs), .structure(let rhs)): - lhs.source == rhs.source && lhs.content == rhs.content - case (.image(let lhs), .image(let rhs)): - lhs.source == rhs.source - default: - false - } - } - } - } - @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @available(watchOS, unavailable) extension SystemLanguageModel { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 3a829982..9365c47c 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -53,18 +53,38 @@ public struct Prompt: Sendable { self.components = components } - func makeTranscriptSegments() throws -> [Transcript.Segment] { - try components.map { component in + func makeTranscriptSegments(occurrenceID: String = UUID().uuidString) throws -> [Transcript.Segment] { + try components.enumerated().map { index, component in + let id = Self.transcriptSegmentID(occurrenceID: occurrenceID, index: index) switch component { case .text(let text): - return .text(.init(content: text.content)) + return .text(.init(id: id, content: text.content)) case .image(_, let content): - return .image(try content.makeSegment(id: UUID().uuidString)) + return .image(try content.makeSegment(id: id)) } } } + func matchesTranscriptOccurrence( + _ segments: [Transcript.Segment], + occurrenceID: String + ) -> Bool { + guard components.count == segments.count else { return false } + return segments.indices.allSatisfy { index in + segments[index].id + == Self.transcriptSegmentID( + occurrenceID: occurrenceID, + index: index + ) + } + } + + private static func transcriptSegmentID(occurrenceID: String, index: Int) -> String { + "\(occurrenceID).\(index)" + } + static func joining(_ prompts: [Prompt], trim: Bool = false) -> Prompt { + guard !prompts.isEmpty else { return Prompt(content: "") } var components: [Component] = [] for prompt in prompts { for component in prompt.components { diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 81a3b3f7..3283aee6 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -43,6 +43,21 @@ struct PromptAttachmentTests { } } + @Test func emptyBuildersKeepOneEmptyTextSegment() throws { + let built = AnyLanguageModel.Prompt {} + let array = AnyLanguageModel.Prompt([AnyLanguageModel.Prompt]()) + + for prompt in [built, array] { + let segments = try prompt.makeTranscriptSegments() + #expect(segments.count == 1) + guard case .text(let text) = segments[0] else { + Issue.record("Empty prompt did not retain its text segment") + continue + } + #expect(text.content.isEmpty) + } + } + @Test func buildersAndRepresentablesPreserveOrderedImages() throws { let original = prompt() struct Representable: AnyLanguageModel.PromptRepresentable { @@ -174,6 +189,42 @@ struct PromptAttachmentTests { == CGImagePropertyOrientation.right.rawValue ) } + + #if canImport(FoundationModels) + @available(macOS 26, iOS 26, visionOS 26, *) + @Test func duplicateSuppressionDoesNotReencodeRecordedAttachments() throws { + let encoded = try AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(try image(), orientation: .right) + }.makeTranscriptSegments() + guard case .image(let image) = try #require(encoded.first), + case .data(let data, _) = image.source + else { + Issue.record("Failed to encode the image fixture") + return + } + + let url = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString) + .appendingPathExtension("png") + try data.write(to: url) + let prompt = AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(imageURL: url, orientation: .right) + } + let occurrenceID = UUID().uuidString + let segments = try prompt.makeTranscriptSegments(occurrenceID: occurrenceID) + try FileManager.default.removeItem(at: url) + let transcript = AnyLanguageModel.Transcript(entries: [ + .prompt(.init(id: occurrenceID, segments: segments)) + ]) + + #expect( + AnyLanguageModel.fmTranscriptDroppingDuplicatePrompt( + transcript, + prompt: prompt + ).isEmpty + ) + } + #endif #endif #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) From 9ab55ea4fe3351a52d1c4d7aefe381058c9faeb4 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 13 Oct 2026 09:10:00 +0800 Subject: [PATCH 10/21] Clarify supported attachment platforms --- Sources/AnyLanguageModel/Models/SystemLanguageModel.swift | 2 +- Tests/AnyLanguageModelTests/PromptAttachmentTests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 2eb7e785..7727bbbf 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -564,7 +564,7 @@ var errorDescription: String? { switch self { case .attachmentsUnavailable: - "Prompt attachments require an OS 27 Foundation Models runtime." + "Prompt attachments require macOS 27, iOS 27, or visionOS 27." case .nonFileImageURL: "Foundation Models prompt attachments require a local file URL." } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 3283aee6..02cf014e 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -231,7 +231,7 @@ struct PromptAttachmentTests { @Test func promptBridgeErrorsHaveLocalizedDescriptions() { #expect( AnyLanguageModel.PromptBridgeError.attachmentsUnavailable.errorDescription - == "Prompt attachments require an OS 27 Foundation Models runtime." + == "Prompt attachments require macOS 27, iOS 27, or visionOS 27." ) #expect( AnyLanguageModel.PromptBridgeError.nonFileImageURL.errorDescription From a03fe05dec40acfa4c85b0f1c61bc7c0e3242219 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 13 Oct 2026 09:08:28 +0800 Subject: [PATCH 11/21] Avoid relowering recorded prompt attachments --- .../LanguageModelSession.swift | 10 ++-- .../Models/SystemLanguageModel.swift | 25 ++------- Sources/AnyLanguageModel/Prompt.swift | 28 ++++++++-- .../PromptAttachmentTests.swift | 51 +++++++++++++++++++ 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index b7775ce3..552c3fe0 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -728,7 +728,7 @@ public final class LanguageModelSession: @unchecked Sendable { let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( id: promptID, - segments: try prompt.makeTranscriptSegments(), + segments: try prompt.makeTranscriptSegments(occurrenceID: promptID), options: options, responseFormat: responseFormat ) @@ -776,14 +776,16 @@ public final class LanguageModelSession: @unchecked Sendable { options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream where Content: Generable { // Add prompt to transcript + let promptID = UUID().uuidString let segments: [Transcript.Segment] do { - segments = try prompt.makeTranscriptSegments() + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) } catch { return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( + id: promptID, segments: segments, options: options, responseFormat: type == String.self ? nil : .init(type: type) @@ -930,14 +932,16 @@ extension LanguageModelSession { includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { + let promptID = UUID().uuidString let segments: [Transcript.Segment] do { - segments = try prompt.makeTranscriptSegments() + segments = try prompt.makeTranscriptSegments(occurrenceID: promptID) } catch { return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) }) } let promptEntry = Transcript.Entry.prompt( Transcript.Prompt( + id: promptID, segments: segments, options: options, responseFormat: .init(schema: schema) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 7354e824..2eb7e785 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -341,32 +341,17 @@ guard let lastEntry = transcript.last, case .prompt(let lastPrompt) = lastEntry else { return transcript } - guard let promptSegments = try? prompt.makeTranscriptSegments(), - lastPrompt.segments.hasSamePayload(as: promptSegments) + guard + prompt.matchesTranscriptOccurrence( + lastPrompt.segments, + occurrenceID: lastPrompt.id + ) else { return transcript } return Transcript(entries: transcript.dropLast()) } - private extension Array where Element == Transcript.Segment { - func hasSamePayload(as other: [Transcript.Segment]) -> Bool { - guard count == other.count else { return false } - return zip(self, other).allSatisfy { lhs, rhs in - switch (lhs, rhs) { - case (.text(let lhs), .text(let rhs)): - lhs.content == rhs.content - case (.structure(let lhs), .structure(let rhs)): - lhs.source == rhs.source && lhs.content == rhs.content - case (.image(let lhs), .image(let rhs)): - lhs.source == rhs.source - default: - false - } - } - } - } - @available(macOS 26.0, iOS 26.0, tvOS 26.0, visionOS 26.0, *) @available(watchOS, unavailable) extension SystemLanguageModel { diff --git a/Sources/AnyLanguageModel/Prompt.swift b/Sources/AnyLanguageModel/Prompt.swift index 3a829982..9365c47c 100644 --- a/Sources/AnyLanguageModel/Prompt.swift +++ b/Sources/AnyLanguageModel/Prompt.swift @@ -53,18 +53,38 @@ public struct Prompt: Sendable { self.components = components } - func makeTranscriptSegments() throws -> [Transcript.Segment] { - try components.map { component in + func makeTranscriptSegments(occurrenceID: String = UUID().uuidString) throws -> [Transcript.Segment] { + try components.enumerated().map { index, component in + let id = Self.transcriptSegmentID(occurrenceID: occurrenceID, index: index) switch component { case .text(let text): - return .text(.init(content: text.content)) + return .text(.init(id: id, content: text.content)) case .image(_, let content): - return .image(try content.makeSegment(id: UUID().uuidString)) + return .image(try content.makeSegment(id: id)) } } } + func matchesTranscriptOccurrence( + _ segments: [Transcript.Segment], + occurrenceID: String + ) -> Bool { + guard components.count == segments.count else { return false } + return segments.indices.allSatisfy { index in + segments[index].id + == Self.transcriptSegmentID( + occurrenceID: occurrenceID, + index: index + ) + } + } + + private static func transcriptSegmentID(occurrenceID: String, index: Int) -> String { + "\(occurrenceID).\(index)" + } + static func joining(_ prompts: [Prompt], trim: Bool = false) -> Prompt { + guard !prompts.isEmpty else { return Prompt(content: "") } var components: [Component] = [] for prompt in prompts { for component in prompt.components { diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 81a3b3f7..3283aee6 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -43,6 +43,21 @@ struct PromptAttachmentTests { } } + @Test func emptyBuildersKeepOneEmptyTextSegment() throws { + let built = AnyLanguageModel.Prompt {} + let array = AnyLanguageModel.Prompt([AnyLanguageModel.Prompt]()) + + for prompt in [built, array] { + let segments = try prompt.makeTranscriptSegments() + #expect(segments.count == 1) + guard case .text(let text) = segments[0] else { + Issue.record("Empty prompt did not retain its text segment") + continue + } + #expect(text.content.isEmpty) + } + } + @Test func buildersAndRepresentablesPreserveOrderedImages() throws { let original = prompt() struct Representable: AnyLanguageModel.PromptRepresentable { @@ -174,6 +189,42 @@ struct PromptAttachmentTests { == CGImagePropertyOrientation.right.rawValue ) } + + #if canImport(FoundationModels) + @available(macOS 26, iOS 26, visionOS 26, *) + @Test func duplicateSuppressionDoesNotReencodeRecordedAttachments() throws { + let encoded = try AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(try image(), orientation: .right) + }.makeTranscriptSegments() + guard case .image(let image) = try #require(encoded.first), + case .data(let data, _) = image.source + else { + Issue.record("Failed to encode the image fixture") + return + } + + let url = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString) + .appendingPathExtension("png") + try data.write(to: url) + let prompt = AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(imageURL: url, orientation: .right) + } + let occurrenceID = UUID().uuidString + let segments = try prompt.makeTranscriptSegments(occurrenceID: occurrenceID) + try FileManager.default.removeItem(at: url) + let transcript = AnyLanguageModel.Transcript(entries: [ + .prompt(.init(id: occurrenceID, segments: segments)) + ]) + + #expect( + AnyLanguageModel.fmTranscriptDroppingDuplicatePrompt( + transcript, + prompt: prompt + ).isEmpty + ) + } + #endif #endif #if canImport(FoundationModels) && compiler(>=6.4) && !os(tvOS) && !os(watchOS) From 51fa0a9707002d39bb4ccf24b5a7eaad56a2e7d6 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 13 Oct 2026 09:10:00 +0800 Subject: [PATCH 12/21] Clarify supported attachment platforms --- Sources/AnyLanguageModel/Models/SystemLanguageModel.swift | 2 +- Tests/AnyLanguageModelTests/PromptAttachmentTests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 2eb7e785..7727bbbf 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -564,7 +564,7 @@ var errorDescription: String? { switch self { case .attachmentsUnavailable: - "Prompt attachments require an OS 27 Foundation Models runtime." + "Prompt attachments require macOS 27, iOS 27, or visionOS 27." case .nonFileImageURL: "Foundation Models prompt attachments require a local file URL." } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 3283aee6..02cf014e 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -231,7 +231,7 @@ struct PromptAttachmentTests { @Test func promptBridgeErrorsHaveLocalizedDescriptions() { #expect( AnyLanguageModel.PromptBridgeError.attachmentsUnavailable.errorDescription - == "Prompt attachments require an OS 27 Foundation Models runtime." + == "Prompt attachments require macOS 27, iOS 27, or visionOS 27." ) #expect( AnyLanguageModel.PromptBridgeError.nonFileImageURL.errorDescription From 1892a39bca496b413e11a7dc0d570381a68e2b1d Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Thu, 15 Oct 2026 09:01:38 +0800 Subject: [PATCH 13/21] Inline Gemini tool output file attachments --- .../Models/GeminiLanguageModel.swift | 47 ++++++++++---- .../GeminiThoughtSignatureTests.swift | 62 +++++++++++++++++++ 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index d531c426..cbb4cf2d 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -254,7 +254,7 @@ public struct GeminiLanguageModel: LanguageModel { requestTranscript.append(entry) } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: try requestTranscript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, @@ -432,7 +432,7 @@ public struct GeminiLanguageModel: LanguageModel { requestTranscript.append(entry) } let params = try createGenerateContentParams( - contents: requestTranscript.toGeminiContent(), + contents: try requestTranscript.toGeminiContent(), tools: geminiTools, generating: type, schema: responseSchema.schema, @@ -800,7 +800,7 @@ private func toGeminiFunctionResponse( ) ) case .url(let url): - parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString))) + parts.append(.inlineData(try geminiFunctionResponseData(from: url))) } } } @@ -816,10 +816,41 @@ private func toGeminiFunctionResponse( ) } +private enum GeminiFunctionResponseAttachmentError: LocalizedError { + case unsupportedURL(URL) + case unsupportedImageType(URL) + + var errorDescription: String? { + switch self { + case .unsupportedURL(let url): + "Gemini Tool output attachments require a local file URL; cannot inline \(url.absoluteString)." + case .unsupportedImageType(let url): + "Gemini Tool output attachments support PNG, JPEG, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)." + } + } +} + +private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineData { + guard url.isFileURL else { + throw GeminiFunctionResponseAttachmentError.unsupportedURL(url) + } + + let mimeType = switch url.pathExtension.lowercased() { + case "png": "image/png" + case "jpg", "jpeg": "image/jpeg" + case "webp": "image/webp" + default: throw GeminiFunctionResponseAttachmentError.unsupportedImageType(url) + } + return try GeminiInlineData( + mimeType: mimeType, + data: Data(contentsOf: url).base64EncodedString() + ) +} + // MARK: - Supporting Types extension Transcript { - fileprivate func toGeminiContent() -> [GeminiContent] { + fileprivate func toGeminiContent() throws -> [GeminiContent] { var messages = [GeminiContent]() for item in self { switch item { @@ -869,13 +900,7 @@ extension Transcript { ) case .toolOutput(let toolOutput): // Add function response as a user message (Gemini API expects function responses from user role) - let functionResponse = - (try? toGeminiFunctionResponse(toolOutput)) - ?? GeminiFunctionResponse( - name: toolOutput.toolName, - response: [:], - parts: nil - ) + let functionResponse = try toGeminiFunctionResponse(toolOutput) messages.append( .init( role: .user, diff --git a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift index d1a93925..006490ab 100644 --- a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift @@ -179,6 +179,68 @@ import Testing #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) != nil) } + @Test("inlines URL-backed images in function responses") + func inlinesURLBackedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + StubURLProtocol.enqueue(json: textResponse("OK")) + + let imageData = Data([0x89, 0x50, 0x4E, 0x47]) + let url = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString) + .appendingPathExtension("png") + try imageData.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let output = Prompt { Attachment(imageURL: url) } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + _ = try await session.respond(to: "read image") + + let body = try #require(StubURLProtocol.recordedBodies.last) + let json = try #require( + try JSONSerialization.jsonObject(with: body) as? [String: Any] + ) + let contents = try #require(json["contents"] as? [[String: Any]]) + let outputParts = try #require(contents.last?["parts"] as? [[String: Any]]) + let functionResponse = try #require( + outputParts.first?["functionResponse"] as? [String: Any] + ) + let mediaParts = try #require(functionResponse["parts"] as? [[String: Any]]) + let inlineData = try #require(mediaParts.first?["inlineData"] as? [String: Any]) + #expect(inlineData["mime_type"] as? String == "image/png") + #expect(Data(base64Encoded: try #require(inlineData["data"] as? String)) == imageData) + #expect(mediaParts.first?["fileData"] == nil) + } + + @Test("rejects remote URL-backed images with an actionable error") + func rejectsRemoteURLBackedToolOutput() async throws { + StubURLProtocol.reset() + StubURLProtocol.enqueue( + json: + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + ) + + let url = try #require(URL(string: "https://example.com/image.png")) + let output = Prompt { Attachment(imageURL: url) } + let session = LanguageModelSession( + model: makeModel(), + tools: [ImageOutputTool(output: output)] + ) + + do { + _ = try await session.respond(to: "read image") + Issue.record("Expected a remote Tool output attachment to be rejected") + } catch { + #expect(error.localizedDescription.contains("local file URL")) + } + } + @Test("does not replay the conversation history on later turns") func doesNotReplayHistoryOnLaterTurns() async throws { StubURLProtocol.reset() From 2f435bd2fd598c7182211b60bac6a0036f6fe498 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Fri, 16 Oct 2026 09:04:44 +0800 Subject: [PATCH 14/21] Format Gemini attachment MIME mapping --- .../Models/GeminiLanguageModel.swift | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index cbb4cf2d..271378c5 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -835,12 +835,13 @@ private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineDat throw GeminiFunctionResponseAttachmentError.unsupportedURL(url) } - let mimeType = switch url.pathExtension.lowercased() { - case "png": "image/png" - case "jpg", "jpeg": "image/jpeg" - case "webp": "image/webp" - default: throw GeminiFunctionResponseAttachmentError.unsupportedImageType(url) - } + let mimeType = + switch url.pathExtension.lowercased() { + case "png": "image/png" + case "jpg", "jpeg": "image/jpeg" + case "webp": "image/webp" + default: throw GeminiFunctionResponseAttachmentError.unsupportedImageType(url) + } return try GeminiInlineData( mimeType: mimeType, data: Data(contentsOf: url).base64EncodedString() From 4d9cd915d377ad1bb02d040548a2e7c67527a4f6 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Sat, 17 Oct 2026 09:00:25 +0800 Subject: [PATCH 15/21] Inline local Anthropic image attachments during transport --- .../Models/AnthropicLanguageModel.swift | 44 +++++++++++++--- .../ProviderUsageTests.swift | 51 +++++++++++++++++++ 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 6a618a79..2fae7d71 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -716,7 +716,7 @@ public struct AnthropicLanguageModel: LanguageModel { .toolResult( .init( toolUseId: invocation.call.id, - content: convertSegmentsToAnthropicContent(invocation.output.segments) + content: try convertSegmentsToAnthropicContent(invocation.output.segments) ) ) ) @@ -1029,14 +1029,14 @@ extension Transcript { messages.append( .init( role: .user, - content: convertSegmentsToAnthropicContent(instructions.segments) + content: try convertSegmentsToAnthropicContent(instructions.segments) ) ) case .prompt(let prompt): messages.append( .init( role: .user, - content: convertSegmentsToAnthropicContent(prompt.segments) + content: try convertSegmentsToAnthropicContent(prompt.segments) ) ) case .reasoning(let reasoning): @@ -1063,7 +1063,7 @@ extension Transcript { case .response(let response): // Anthropic rejects text blocks without non-whitespace text, // such as the empty response of a turn that only called tools. - let content = convertSegmentsToAnthropicContent(response.segments).filter { block in + let content = try convertSegmentsToAnthropicContent(response.segments).filter { block in guard case .text(let text) = block else { return true } return !text.text.allSatisfy(\.isWhitespace) } @@ -1091,7 +1091,7 @@ extension Transcript { .toolResult( AnthropicToolResult( toolUseId: toolOutput.id, - content: convertSegmentsToAnthropicContent(toolOutput.segments) + content: try convertSegmentsToAnthropicContent(toolOutput.segments) ) ) ] @@ -1244,7 +1244,7 @@ private extension AnthropicThinking { } } -private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) -> [AnthropicContent] { +private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) throws -> [AnthropicContent] { var blocks: [AnthropicContent] = [] blocks.reserveCapacity(segments.count) for segment in segments { @@ -1256,7 +1256,26 @@ private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) case .image(let img): switch img.source { case .url(let url): - blocks.append(.image(AnthropicImage(url: url.absoluteString))) + if url.isFileURL { + let mimeType: String + switch url.pathExtension.lowercased() { + case "jpg", "jpeg": mimeType = "image/jpeg" + case "png": mimeType = "image/png" + case "gif": mimeType = "image/gif" + case "webp": mimeType = "image/webp" + default: throw AnthropicImageAttachmentError.unsupportedImageType(url) + } + blocks.append( + .image( + AnthropicImage( + base64Data: try Data(contentsOf: url).base64EncodedString(), + mimeType: mimeType + ) + ) + ) + } else { + blocks.append(.image(AnthropicImage(url: url.absoluteString))) + } case .data(let data, let mimeType): blocks.append(.image(AnthropicImage(base64Data: data.base64EncodedString(), mimeType: mimeType))) } @@ -1265,6 +1284,17 @@ private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) return blocks } +private enum AnthropicImageAttachmentError: LocalizedError { + case unsupportedImageType(URL) + + var errorDescription: String? { + switch self { + case .unsupportedImageType(let url): + "Anthropic image attachments support PNG, JPEG, GIF, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)." + } + } +} + private struct AnthropicToolUse: Codable, Sendable { let type: String let id: String diff --git a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift index c4331da1..a2f9cc7b 100644 --- a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift +++ b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift @@ -482,6 +482,57 @@ import Testing #expect(UsageURLProtocol.recordedBodies.count == 2) } + @Test(arguments: [false, true]) + func anthropicInlinesLocalToolImagesAndReplaysThem(_ streaming: Bool) async throws { + let provider = Provider.anthropic + UsageURLProtocol.reset() + UsageURLProtocol.enqueue( + json: try streaming ? provider.toolStream() : Self.json(provider.response(text: "", tool: true)) + ) + if streaming { + UsageURLProtocol.enqueue(json: try provider.stream()) + } + UsageURLProtocol.enqueue(json: try Self.json(provider.response())) + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension("png") + try testImageData.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let session = provider.makeSession(tools: [LocalImageWeatherTool(url: url)]) + + if streaming { + _ = try await session.streamResponse(to: "Weather?").collect() + } else { + _ = try await session.respond(to: "Weather?") + } + _ = try await session.respond(to: "Review the image") + + #expect(UsageURLProtocol.recordedBodies.count == (streaming ? 3 : 2)) + for body in UsageURLProtocol.recordedBodies.dropFirst() { + let request = try JSONDecoder().decode([String: JSONValue].self, from: body) + let messages = try #require(request["messages"]?.arrayValue) + let results = messages.flatMap { $0.objectValue?["content"]?.arrayValue ?? [] } + .filter { $0.objectValue?["type"] == .string("tool_result") } + let content = try #require(results.first?.objectValue?["content"]?.arrayValue) + let image = try #require(content.first { $0.objectValue?["type"] == .string("image") }) + let source = try #require(image.objectValue?["source"]?.objectValue) + #expect(source["type"] == .string("base64")) + #expect(source["media_type"] == .string("image/png")) + #expect(source["data"] == .string(testImageData.base64EncodedString())) + #expect(source["url"] == nil) + } + } + + private struct LocalImageWeatherTool: Tool { + let name = "getWeather" + let description = "Get weather as an image" + let url: URL + + func call(arguments: WeatherTool.Arguments) async throws -> Prompt { + Prompt { Attachment(imageURL: url) } + } + } + @Test(arguments: Provider.allCases) func stoppedStreamedTools(_ provider: Provider) async throws { UsageURLProtocol.reset() From 5a17ad3c1079afcdf4eca5efbd561e8b93bccd3f Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Sat, 17 Oct 2026 23:33:20 +0800 Subject: [PATCH 16/21] Snapshot local tool image data before persisting output --- README.md | 5 ++ Sources/AnyLanguageModel/Tool.swift | 40 ++++++++++++++- .../PromptAttachmentTests.swift | 27 +++++++++- .../ProviderUsageTests.swift | 50 +++++++++++++++---- 4 files changed, 110 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a4a871e3..0e4e63cb 100644 --- a/README.md +++ b/README.md @@ -409,6 +409,11 @@ they throw an error when a prompt has an attachment. The system bridge requires local file URLs; other provider adapters retain URL images in their existing transcript representation. +Ordinary Tool outputs read local image files once when the output is created, +and retain the image bytes and MIME type in the compatibility transcript. +Continuations and saved transcript replay therefore do not depend on the original +file remaining at its URL. Remote image URLs are retained without downloading. + This compatibility surface covers `CGImage` and `imageURL` attachments in prompts and ordinary Tool outputs. Instruction attachments, labels, `CIImage`, pixel buffers, and newer data attachments remain outside this change. diff --git a/Sources/AnyLanguageModel/Tool.swift b/Sources/AnyLanguageModel/Tool.swift index 59f5b800..558bb628 100644 --- a/Sources/AnyLanguageModel/Tool.swift +++ b/Sources/AnyLanguageModel/Tool.swift @@ -1,3 +1,9 @@ +import Foundation + +#if canImport(UniformTypeIdentifiers) + import UniformTypeIdentifiers +#endif + /// A tool that a model can call to gather information at runtime or perform side effects. /// /// Tool calling gives the model the ability to call your code to incorporate @@ -127,6 +133,38 @@ extension Tool { return [Transcript.Segment.text(.init(content: stringOutput))] } - return try output.promptRepresentation.makeTranscriptSegments() + return try output.promptRepresentation.makeTranscriptSegments().map { segment in + guard case .image(let image) = segment, + case .url(let url) = image.source, + url.isFileURL + else { return segment } + + let mimeType: String? + #if canImport(UniformTypeIdentifiers) + mimeType = UTType(filenameExtension: url.pathExtension)?.preferredMIMEType + #else + mimeType = + switch url.pathExtension.lowercased() { + case "jpg", "jpeg": "image/jpeg" + case "png": "image/png" + case "gif": "image/gif" + case "webp": "image/webp" + default: nil + } + #endif + guard let mimeType else { throw ToolAttachmentError.unknownImageType(url) } + return .image(.init(id: image.id, data: try Data(contentsOf: url), mimeType: mimeType)) + } + } +} + +private enum ToolAttachmentError: LocalizedError { + case unknownImageType(URL) + + var errorDescription: String? { + switch self { + case .unknownImageType(let url): + "Cannot determine the MIME type of the Tool output image \(url.lastPathComponent)." + } } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 589ce23f..77f1dd6b 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -123,7 +123,15 @@ struct PromptAttachmentTests { } @Test func ordinaryToolReturnsOrderedImageSegments() async throws { - let segments = try await ImageTool(output: prompt()).makeOutputSegments( + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".png") + try testImageData.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let output = AnyLanguageModel.Prompt { + "before" + AnyLanguageModel.Attachment(imageURL: url) + "after" + } + let segments = try await ImageTool(output: output).makeOutputSegments( from: AnyLanguageModel.GeneratedContent("read") ) #expect(segments.count == 3) @@ -136,7 +144,8 @@ struct PromptAttachmentTests { } #expect(before.content == "before") #expect(after.content == "after") - #expect(image.source == .url(imageURL)) + #expect(image.source == .data(testImageData, mimeType: "image/png")) + try FileManager.default.removeItem(at: url) let transcript = AnyLanguageModel.Transcript(entries: [ .toolOutput(.init(id: "image-call", toolName: "image", segments: segments)) @@ -145,6 +154,20 @@ struct PromptAttachmentTests { #expect(try JSONDecoder().decode(AnyLanguageModel.Transcript.self, from: encoded) == transcript) } + @Test func ordinaryToolRetainsRemoteImageURLs() async throws { + let url = try #require(URL(string: "https://example.com/image.png")) + let segments = try await ImageTool( + output: AnyLanguageModel.Prompt { + AnyLanguageModel.Attachment(imageURL: url) + } + ).makeOutputSegments(from: AnyLanguageModel.GeneratedContent("read")) + guard case .image(let image) = try #require(segments.first) else { + Issue.record("Missing image") + return + } + #expect(image.source == .url(url)) + } + #if canImport(CoreGraphics) && canImport(ImageIO) private func image() throws -> CGImage { let context = try #require( diff --git a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift index f1252f4b..b0215f99 100644 --- a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift +++ b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift @@ -9,7 +9,7 @@ import Testing enum Provider: CaseIterable, Equatable, Sendable { case chat, responses, openResponses, anthropic, gemini, ollama - func makeSession(tools: [any Tool] = []) -> LanguageModelSession { + func makeSession(tools: [any Tool] = [], transcript: Transcript = .init()) -> LanguageModelSession { let http = UsageURLProtocol.makeSession() let model: any LanguageModel switch self { @@ -34,7 +34,7 @@ import Testing case .ollama: model = OllamaLanguageModel(model: "test", session: http) } - return LanguageModelSession(model: model, tools: tools) + return LanguageModelSession(model: model, tools: tools, transcript: transcript) } var counts: [String: Any] { @@ -482,15 +482,16 @@ import Testing #expect(UsageURLProtocol.recordedBodies.count == 2) } - @Test(arguments: [false, true]) - func anthropicInlinesLocalToolImagesAndReplaysThem(_ streaming: Bool) async throws { - let provider = Provider.anthropic + @Test(arguments: [Provider.anthropic, .gemini], [false, true]) + func localToolImagesSurviveDeletionAndTranscriptRestoration(_ provider: Provider, _ streaming: Bool) + async throws + { UsageURLProtocol.reset() UsageURLProtocol.enqueue( json: try streaming ? provider.toolStream() : Self.json(provider.response(text: "", tool: true)) ) - if streaming { - UsageURLProtocol.enqueue(json: try provider.stream()) + if streaming || provider == .gemini { + UsageURLProtocol.enqueue(json: try streaming ? provider.stream() : Self.json(provider.response())) } UsageURLProtocol.enqueue(json: try Self.json(provider.response())) let url = FileManager.default.temporaryDirectory @@ -499,17 +500,31 @@ import Testing try testImageData.write(to: url) defer { try? FileManager.default.removeItem(at: url) } let session = provider.makeSession(tools: [LocalImageWeatherTool(url: url)]) + session.toolExecutionDelegate = DeleteImageAfterToolOutput(url: url) if streaming { _ = try await session.streamResponse(to: "Weather?").collect() } else { _ = try await session.respond(to: "Weather?") } - _ = try await session.respond(to: "Review the image") + #expect(!FileManager.default.fileExists(atPath: url.path)) + let archived = try JSONEncoder().encode(session.transcript) + let restored = provider.makeSession(transcript: try JSONDecoder().decode(Transcript.self, from: archived)) + _ = try await restored.respond(to: "Review the image") - #expect(UsageURLProtocol.recordedBodies.count == (streaming ? 3 : 2)) + #expect(UsageURLProtocol.recordedBodies.count == (streaming || provider == .gemini ? 3 : 2)) for body in UsageURLProtocol.recordedBodies.dropFirst() { let request = try JSONDecoder().decode([String: JSONValue].self, from: body) + if provider == .gemini { + let contents = try #require(request["contents"]?.arrayValue) + let results = contents.flatMap { $0.objectValue?["parts"]?.arrayValue ?? [] } + .compactMap { $0.objectValue?["functionResponse"]?.objectValue } + let parts = try #require(results.first?["parts"]?.arrayValue) + let inlineData = try #require(parts.first?.objectValue?["inlineData"]?.objectValue) + #expect(inlineData["mime_type"] == .string("image/png")) + #expect(inlineData["data"] == .string(testImageData.base64EncodedString())) + continue + } let messages = try #require(request["messages"]?.arrayValue) let results = messages.flatMap { $0.objectValue?["content"]?.arrayValue ?? [] } .filter { $0.objectValue?["type"] == .string("tool_result") } @@ -523,6 +538,23 @@ import Testing } } + private struct DeleteImageAfterToolOutput: ToolExecutionDelegate { + let url: URL + + func didExecuteToolCall( + _ toolCall: Transcript.ToolCall, + output: Transcript.ToolOutput, + in session: LanguageModelSession + ) async { + guard case .image(let image)? = output.segments.first else { + Issue.record("Missing image in Tool output") + return + } + #expect(image.source == .data(testImageData, mimeType: "image/png")) + do { try FileManager.default.removeItem(at: url) } catch { Issue.record(error) } + } + } + private struct LocalImageWeatherTool: Tool { let name = "getWeather" let description = "Get weather as an image" From cf756b173da31011c9419ac7d751a099d94f870f Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Mon, 19 Oct 2026 09:18:11 +0800 Subject: [PATCH 17/21] Preserve native Tool attachment snapshots through delegate execution --- .../Models/SystemLanguageModel.swift | 16 ++++--- .../SystemLanguageModelTests.swift | 44 +++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index 88a5e632..4eb80a16 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -744,9 +744,10 @@ Transcript.ToolOutput(id: call.id, toolName: tool.name, segments: segments) } guard let session, let delegate = session.toolExecutionDelegate else { - let text = try await tool.callFunction(content: arguments).promptRepresentation.description - await ToolCallRecord.current?.ran(call, output: output([.text(.init(content: text))])) - return FoundationModels.Prompt(text) + let segments = try await tool.makeOutputSegments(from: arguments) + let prompt = try segments.toFoundationModelsToolOutput() + await ToolCallRecord.current?.ran(call, output: output(segments)) + return prompt } await delegate.didGenerateToolCalls([call], in: session) switch await delegate.toolCallDecision(for: call, in: session) { @@ -761,10 +762,11 @@ return prompt case .execute: do { - let text = try await tool.callFunction(content: arguments).promptRepresentation.description - await delegate.didExecuteToolCall(call, output: output([.text(.init(content: text))]), in: session) - await ToolCallRecord.current?.ran(call, output: output([.text(.init(content: text))])) - return FoundationModels.Prompt(text) + let segments = try await tool.makeOutputSegments(from: arguments) + let prompt = try segments.toFoundationModelsToolOutput() + await delegate.didExecuteToolCall(call, output: output(segments), in: session) + await ToolCallRecord.current?.ran(call, output: output(segments)) + return prompt } catch { await delegate.didFailToolCall(call, error: error, in: session) throw error diff --git a/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift b/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift index dbee954d..4a70475e 100644 --- a/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/SystemLanguageModelTests.swift @@ -323,6 +323,50 @@ import Testing } #if compiler(>=6.4) + @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) + @Test(arguments: [false, true]) + func ordinaryNativeToolImageSurvivesDeletionAndReplay(useDelegate: Bool) async throws { + let data = try onePixelPNG() + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + .appendingPathExtension("png") + try data.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let tool = NativeImageOutputTool(imageURL: url) + let session = LanguageModelSession(model: SystemLanguageModel.default, tools: [tool]) + if useDelegate { session.toolExecutionDelegate = ScriptedDelegate([.execute]) } + let record = ToolCallRecord() + try await ToolCallRecord.$current.withValue(record) { + _ = try await callTool(tool, with: GeneratedContent("image"), in: session) + } + try FileManager.default.removeItem(at: url) + let transcript = Transcript(entries: await record.entries) + let replay = try JSONDecoder().decode(Transcript.self, from: JSONEncoder().encode(transcript)) + let outputs = replay.compactMap { entry -> Transcript.ToolOutput? in + if case .toolOutput(let output) = entry { return output } + return nil + } + let output = try #require(outputs.first) + #expect(output.segments.count == 2) + guard case .image(let image) = output.segments.last else { + Issue.record("Expected the native Tool's image, not its text description") + return + } + #expect(image.source == .data(data, mimeType: "image/png")) + _ = try output.segments.toFoundationModelsToolOutput() + } + + private struct NativeImageOutputTool: Tool { + let imageURL: URL + let description = "Return an image" + + func call(arguments: String) async throws -> Prompt { + Prompt { + "Image output" + Attachment(imageURL: imageURL) + } + } + } + /// An image the delegate provides goes to Foundation Models as an attachment. @available(macOS 27.0, iOS 27.0, visionOS 27.0, *) @Test func providedImageIsAttached() async throws { From c96bd09eac92dfaeb9c8a2710b0fc192bb8d3841 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 20 Oct 2026 09:21:20 +0800 Subject: [PATCH 18/21] Reject non-image file types when snapshotting Tool attachments --- Sources/AnyLanguageModel/Tool.swift | 11 ++++++++++- .../PromptAttachmentTests.swift | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Sources/AnyLanguageModel/Tool.swift b/Sources/AnyLanguageModel/Tool.swift index 558bb628..d72ef9ca 100644 --- a/Sources/AnyLanguageModel/Tool.swift +++ b/Sources/AnyLanguageModel/Tool.swift @@ -141,7 +141,13 @@ extension Tool { let mimeType: String? #if canImport(UniformTypeIdentifiers) - mimeType = UTType(filenameExtension: url.pathExtension)?.preferredMIMEType + guard let type = UTType(filenameExtension: url.pathExtension) else { + throw ToolAttachmentError.unknownImageType(url) + } + guard type.conforms(to: .image) else { + throw ToolAttachmentError.unsupportedImageType(url) + } + mimeType = type.preferredMIMEType #else mimeType = switch url.pathExtension.lowercased() { @@ -160,11 +166,14 @@ extension Tool { private enum ToolAttachmentError: LocalizedError { case unknownImageType(URL) + case unsupportedImageType(URL) var errorDescription: String? { switch self { case .unknownImageType(let url): "Cannot determine the MIME type of the Tool output image \(url.lastPathComponent)." + case .unsupportedImageType(let url): + "The Tool output attachment \(url.lastPathComponent) is not an image file." } } } diff --git a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift index 77f1dd6b..348c93e3 100644 --- a/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift +++ b/Tests/AnyLanguageModelTests/PromptAttachmentTests.swift @@ -154,6 +154,18 @@ struct PromptAttachmentTests { #expect(try JSONDecoder().decode(AnyLanguageModel.Transcript.self, from: encoded) == transcript) } + @Test(arguments: ["pdf", "txt"]) + func ordinaryToolRejectsNonImageFiles(fileExtension: String) async throws { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + .appendingPathExtension(fileExtension) + try Data("Not an image".utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + await #expect(throws: (any Error).self) { + _ = try await ImageTool(output: AnyLanguageModel.Prompt { AnyLanguageModel.Attachment(imageURL: url) }) + .makeOutputSegments(from: AnyLanguageModel.GeneratedContent("read")) + } + } + @Test func ordinaryToolRetainsRemoteImageURLs() async throws { let url = try #require(URL(string: "https://example.com/image.png")) let segments = try await ImageTool( From 07ad9540f1d95d68ed842304971c88eb96e18733 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Tue, 20 Oct 2026 13:08:57 +0800 Subject: [PATCH 19/21] Describe remote Gemini Tool images as URL text --- README.md | 4 + .../Models/GeminiLanguageModel.swift | 6 +- .../GeminiThoughtSignatureTests.swift | 79 ++++++++++++++++--- 3 files changed, 75 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1632bff6..24c7a42e 100644 --- a/README.md +++ b/README.md @@ -414,6 +414,10 @@ and retain the image bytes and MIME type in the compatibility transcript. Continuations and saved transcript replay therefore do not depend on the original file remaining at its URL. Remote image URLs are retained without downloading. +Gemini supports images in Tool function responses only with Gemini 3 and later. +In Gemini Tool outputs, remote image URLs are sent as text at their original +position without downloading the image; the transcript retains the image segment. + This compatibility surface covers `CGImage` and `imageURL` attachments in prompts and ordinary Tool outputs. Instruction attachments, labels, `CIImage`, pixel buffers, and newer data attachments remain outside this change. diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index c9ada7fd..cfbbc455 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -814,7 +814,11 @@ private func toGeminiFunctionResponse( ) ) case .url(let url): - parts.append(.inlineData(try geminiFunctionResponseData(from: url))) + if url.isFileURL { + parts.append(.inlineData(try geminiFunctionResponseData(from: url))) + } else { + resultValues.append(url.absoluteString) + } } } } diff --git a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift index 006490ab..6ee026b6 100644 --- a/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiThoughtSignatureTests.swift @@ -218,27 +218,80 @@ import Testing #expect(mediaParts.first?["fileData"] == nil) } - @Test("rejects remote URL-backed images with an actionable error") - func rejectsRemoteURLBackedToolOutput() async throws { + @Test("sends remote Tool image URLs as ordered text without downloading", arguments: [false, true]) + func describesRemoteURLBackedToolOutput(streaming: Bool) async throws { StubURLProtocol.reset() - StubURLProtocol.enqueue( - json: - #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# - ) + let toolResponse = + #"{"candidates":[{"content":{"role":"model","parts":[{"functionCall":{"name":"image","args":{"request":"read"}}}]},"finishReason":"STOP"}]}"# + if streaming { + StubURLProtocol.enqueue(eventStream: [toolResponse]) + StubURLProtocol.enqueue(eventStream: [try self.response(parts: [["text": "OK"]])]) + } else { + StubURLProtocol.enqueue(json: toolResponse) + StubURLProtocol.enqueue(json: textResponse("OK")) + } - let url = try #require(URL(string: "https://example.com/image.png")) - let output = Prompt { Attachment(imageURL: url) } + let url = try #require(URL(string: "https://example.invalid/image.png")) + let output = Prompt { + "before" + Attachment(imageURL: url) + "after" + } let session = LanguageModelSession( model: makeModel(), tools: [ImageOutputTool(output: output)] ) - do { - _ = try await session.respond(to: "read image") - Issue.record("Expected a remote Tool output attachment to be rejected") - } catch { - #expect(error.localizedDescription.contains("local file URL")) + let response = + if streaming { + try await session.streamResponse(to: "read image").collect() + } else { + try await session.respond(to: "read image") + } + #expect(response.content == "OK") + let bodies = StubURLProtocol.recordedBodies + try #require(bodies.count == 2) + + func expectURLText(in body: Data) throws { + let responses = try contents(in: body).flatMap { content in + (content["parts"] as? [[String: Any]] ?? []).compactMap { + $0["functionResponse"] as? [String: Any] + } + } + try #require(responses.count == 1) + let functionResponse = try #require(responses.first) + let result = try #require(functionResponse["response"] as? [String: Any]) + #expect(result["result"] as? String == "before\n\(url.absoluteString)\nafter") + #expect(functionResponse["parts"] == nil) + } + try expectURLText(in: bodies[1]) + + let outputs = session.transcript.compactMap { entry -> Transcript.ToolOutput? in + if case .toolOutput(let output) = entry { return output } + return nil + } + let segments = try #require(outputs.first).segments + try #require(segments.count == 3) + guard case .image(let image) = segments[1] else { + Issue.record("The transcript must retain the remote image segment") + return + } + #expect(image.source == .url(url)) + + let data = try JSONEncoder().encode(session.transcript) + let transcript = try JSONDecoder().decode(Transcript.self, from: data) + #expect(transcript == session.transcript) + let restored = LanguageModelSession(model: makeModel(), transcript: transcript) + if streaming { + StubURLProtocol.enqueue(eventStream: [try self.response(parts: [["text": "Replayed"]])]) + #expect(try await restored.streamResponse(to: "continue").collect().content == "Replayed") + } else { + StubURLProtocol.enqueue(json: textResponse("Replayed")) + #expect(try await restored.respond(to: "continue").content == "Replayed") } + let replayBodies = StubURLProtocol.recordedBodies + try #require(replayBodies.count == 3) + try expectURLText(in: replayBodies[2]) } @Test("does not replay the conversation history on later turns") From 14d1b74f91bd18bd46a6656fc3da5f3341005f91 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Wed, 21 Oct 2026 09:32:15 +0800 Subject: [PATCH 20/21] Remove unreachable remote-URL failure from Gemini file lowering --- Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index cfbbc455..c2eebe69 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -835,13 +835,10 @@ private func toGeminiFunctionResponse( } private enum GeminiFunctionResponseAttachmentError: LocalizedError { - case unsupportedURL(URL) case unsupportedImageType(URL) var errorDescription: String? { switch self { - case .unsupportedURL(let url): - "Gemini Tool output attachments require a local file URL; cannot inline \(url.absoluteString)." case .unsupportedImageType(let url): "Gemini Tool output attachments support PNG, JPEG, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)." } @@ -849,10 +846,6 @@ private enum GeminiFunctionResponseAttachmentError: LocalizedError { } private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineData { - guard url.isFileURL else { - throw GeminiFunctionResponseAttachmentError.unsupportedURL(url) - } - let mimeType = switch url.pathExtension.lowercased() { case "png": "image/png" From 77e57d07a53ccb7a3de74ccea969c000442f3cc7 Mon Sep 17 00:00:00 2001 From: "Ronnie W." Date: Wed, 21 Oct 2026 10:00:21 +0800 Subject: [PATCH 21/21] Use image placeholders for unsupported provider tool-output formats --- .../Models/AnthropicLanguageModel.swift | 26 +- .../Models/GeminiLanguageModel.swift | 35 ++- .../ProviderUsageTests.swift | 248 +++++++++++++++++- 3 files changed, 285 insertions(+), 24 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 4087241d..3f7ac123 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -723,7 +723,10 @@ public struct AnthropicLanguageModel: LanguageModel { .toolResult( .init( toolUseId: invocation.call.id, - content: try convertSegmentsToAnthropicContent(invocation.output.segments) + content: try convertSegmentsToAnthropicContent( + invocation.output.segments, + isToolOutput: true + ) ) ) ) @@ -1142,7 +1145,10 @@ extension Transcript { .toolResult( AnthropicToolResult( toolUseId: toolOutput.id, - content: try convertSegmentsToAnthropicContent(toolOutput.segments) + content: try convertSegmentsToAnthropicContent( + toolOutput.segments, + isToolOutput: true + ) ) ) ] @@ -1295,7 +1301,10 @@ private extension AnthropicThinking { } } -private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) throws -> [AnthropicContent] { +private func convertSegmentsToAnthropicContent( + _ segments: [Transcript.Segment], + isToolOutput: Bool = false +) throws -> [AnthropicContent] { var blocks: [AnthropicContent] = [] blocks.reserveCapacity(segments.count) for segment in segments { @@ -1314,7 +1323,12 @@ private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) case "png": mimeType = "image/png" case "gif": mimeType = "image/gif" case "webp": mimeType = "image/webp" - default: throw AnthropicImageAttachmentError.unsupportedImageType(url) + default: + if isToolOutput { + blocks.append(.text(AnthropicText(text: ""))) + continue + } + throw AnthropicImageAttachmentError.unsupportedImageType(url) } blocks.append( .image( @@ -1328,6 +1342,10 @@ private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) blocks.append(.image(AnthropicImage(url: url.absoluteString))) } case .data(let data, let mimeType): + if isToolOutput && !["image/jpeg", "image/png", "image/gif", "image/webp"].contains(mimeType) { + blocks.append(.text(AnthropicText(text: ""))) + continue + } blocks.append(.image(AnthropicImage(base64Data: data.base64EncodedString(), mimeType: mimeType))) } } diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index c2eebe69..d67499ec 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -805,6 +805,10 @@ private func toGeminiFunctionResponse( case .image(let image): switch image.source { case .data(let data, let mimeType): + guard ["image/png", "image/jpeg", "image/webp"].contains(mimeType) else { + resultValues.append("") + continue + } parts.append( .inlineData( GeminiInlineData( @@ -815,7 +819,11 @@ private func toGeminiFunctionResponse( ) case .url(let url): if url.isFileURL { - parts.append(.inlineData(try geminiFunctionResponseData(from: url))) + if let data = try geminiFunctionResponseData(from: url) { + parts.append(.inlineData(data)) + } else { + resultValues.append("") + } } else { resultValues.append(url.absoluteString) } @@ -834,25 +842,14 @@ private func toGeminiFunctionResponse( ) } -private enum GeminiFunctionResponseAttachmentError: LocalizedError { - case unsupportedImageType(URL) - - var errorDescription: String? { - switch self { - case .unsupportedImageType(let url): - "Gemini Tool output attachments support PNG, JPEG, and WebP files; cannot determine a supported MIME type for \(url.lastPathComponent)." - } +private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineData? { + let mimeType: String + switch url.pathExtension.lowercased() { + case "png": mimeType = "image/png" + case "jpg", "jpeg": mimeType = "image/jpeg" + case "webp": mimeType = "image/webp" + default: return nil } -} - -private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineData { - let mimeType = - switch url.pathExtension.lowercased() { - case "png": "image/png" - case "jpg", "jpeg": "image/jpeg" - case "webp": "image/webp" - default: throw GeminiFunctionResponseAttachmentError.unsupportedImageType(url) - } return try GeminiInlineData( mimeType: mimeType, data: Data(contentsOf: url).base64EncodedString() diff --git a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift index b40134e8..54604ca6 100644 --- a/Tests/AnyLanguageModelTests/ProviderUsageTests.swift +++ b/Tests/AnyLanguageModelTests/ProviderUsageTests.swift @@ -538,8 +538,254 @@ import Testing } } + @Test(arguments: [Provider.anthropic, .gemini], [false, true]) + func toolOutputImageMIMEsPreserveOnlySupportedWireParts(_ provider: Provider, _ streaming: Bool) async throws { + for mimeType in [ + "image/png", "image/jpeg", "image/gif", "image/webp", "image/tiff", "image/svg+xml", "image/avif", + "application/octet-stream", + ] { + let supported = + ["image/png", "image/jpeg", "image/webp"].contains(mimeType) + || (provider == .anthropic && mimeType == "image/gif") + let segments: [Transcript.Segment] = [ + .text(.init(content: "before")), + .image(.init(data: testImageData, mimeType: mimeType)), + .text(.init(content: "after")), + ] + let session = provider.makeSession(tools: [WeatherTool()]) + session.toolExecutionDelegate = ImageSegmentsDelegate(segments: segments) + try await checkImageToolContinuationAndReplay( + provider, + streaming: streaming, + session: session, + expectedTexts: supported ? ["before", "after"] : ["before", "", "after"], + expectedMIME: supported ? mimeType : nil + ) + let output = try #require( + session.transcript.compactMap { entry -> Transcript.ToolOutput? in + if case .toolOutput(let output) = entry { return output } + return nil + }.first + ) + #expect(output.segments == segments) + } + } + + @Test(arguments: [Provider.anthropic, .gemini], [false, true]) + func localToolImageFormatsSnapshotBeforePlaceholder(_ provider: Provider, _ streaming: Bool) async throws { + for (fileExtension, mimeType) in [("tiff", "image/tiff"), ("svg", "image/svg+xml"), ("gif", "image/gif")] { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString).appendingPathExtension(fileExtension) + try testImageData.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let session = provider.makeSession(tools: [LocalImageWeatherTool(url: url)]) + session.toolExecutionDelegate = DeleteImageAfterToolOutput(url: url, mimeType: mimeType) + let supported = provider == .anthropic && fileExtension == "gif" + try await checkImageToolContinuationAndReplay( + provider, + streaming: streaming, + session: session, + expectedTexts: supported ? [] : [""], + expectedMIME: supported ? mimeType : nil + ) + #expect(!FileManager.default.fileExists(atPath: url.path)) + let output = try #require( + session.transcript.compactMap { entry -> Transcript.ToolOutput? in + if case .toolOutput(let output) = entry { return output } + return nil + }.first + ) + guard case .image(let image)? = output.segments.first else { + Issue.record("Wire placeholder must not replace the canonical image") + continue + } + #expect(image.source == .data(testImageData, mimeType: mimeType)) + } + } + + private func checkImageToolContinuationAndReplay( + _ provider: Provider, + streaming: Bool, + session: LanguageModelSession, + expectedTexts: [String], + expectedMIME: String? + ) async throws { + UsageURLProtocol.reset() + UsageURLProtocol.enqueue( + json: try streaming ? provider.toolStream() : Self.json(provider.response(text: "", tool: true)) + ) + if streaming || provider == .gemini { + UsageURLProtocol.enqueue(json: try streaming ? provider.stream() : Self.json(provider.response())) + } + if streaming { + _ = try await session.streamResponse(to: "Weather?").collect() + } else { + _ = try await session.respond(to: "Weather?") + } + let archived = try JSONEncoder().encode(session.transcript) + let transcript = try JSONDecoder().decode(Transcript.self, from: archived) + #expect(transcript == session.transcript) + let restored = provider.makeSession(transcript: transcript) + UsageURLProtocol.enqueue(json: try streaming ? provider.stream() : Self.json(provider.response())) + if streaming { + _ = try await restored.streamResponse(to: "Review the image").collect() + } else { + _ = try await restored.respond(to: "Review the image") + } + #expect(UsageURLProtocol.recordedBodies.count == (streaming || provider == .gemini ? 3 : 2)) + for body in UsageURLProtocol.recordedBodies.dropFirst() { + try expectToolImage(in: body, provider: provider, texts: expectedTexts, mimeType: expectedMIME) + } + } + + private func expectToolImage(in body: Data, provider: Provider, texts: [String], mimeType: String?) throws { + let request = try JSONDecoder().decode([String: JSONValue].self, from: body) + if provider == .gemini { + let contents = try #require(request["contents"]?.arrayValue) + let results = contents.flatMap { $0.objectValue?["parts"]?.arrayValue ?? [] } + .compactMap { $0.objectValue?["functionResponse"]?.objectValue } + try #require(results.count == 1) + let result = try #require(results.first) + #expect( + result["response"]?.objectValue?["result"] + == (texts.isEmpty ? nil : .string(texts.joined(separator: "\n"))) + ) + let parts = result["parts"]?.arrayValue ?? [] + #expect(parts.count == (mimeType == nil ? 0 : 1)) + if let mimeType { + let image = try #require(parts.first?.objectValue?["inlineData"]?.objectValue) + #expect(image["mime_type"] == .string(mimeType)) + #expect(image["data"] == .string(testImageData.base64EncodedString())) + } + } else { + let messages = try #require(request["messages"]?.arrayValue) + let results = messages.flatMap { $0.objectValue?["content"]?.arrayValue ?? [] } + .filter { $0.objectValue?["type"] == .string("tool_result") } + try #require(results.count == 1) + let content = try #require(results.first?.objectValue?["content"]?.arrayValue) + #expect(content.compactMap { $0.objectValue?["text"]?.stringValue } == texts) + let images = content.filter { $0.objectValue?["type"] == .string("image") } + #expect(images.count == (mimeType == nil ? 0 : 1)) + if let mimeType { + let source = try #require(images.first?.objectValue?["source"]?.objectValue) + #expect(source["media_type"] == .string(mimeType)) + #expect(source["data"] == .string(testImageData.base64EncodedString())) + #expect(source["type"] == .string("base64")) + #expect( + content.map { $0.objectValue?["type"]?.stringValue } + == (texts.isEmpty ? ["image"] : ["text", "image", "text"]) + ) + } else { + #expect(content.allSatisfy { $0.objectValue?["type"] == .string("text") }) + } + } + } + + private struct ImageSegmentsDelegate: ToolExecutionDelegate { + let segments: [Transcript.Segment] + + func toolCallDecision(for toolCall: Transcript.ToolCall, in session: LanguageModelSession) async + -> ToolExecutionDecision + { .provideOutput(segments) } + } + + @Test(arguments: [Provider.anthropic, .gemini]) + func legacyUnsupportedToolImageURLsUsePlaceholder(_ provider: Provider) async throws { + for fileExtension in ["tiff", "svg"] { + UsageURLProtocol.reset() + UsageURLProtocol.enqueue(json: try Self.json(provider.response())) + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString).appendingPathExtension(fileExtension) + let transcript = Transcript(entries: [ + .toolCalls( + .init([.init(id: "call_1", toolName: "getWeather", arguments: GeneratedContent("Paris"))]) + ), + .toolOutput( + .init(id: "call_1", toolName: "getWeather", segments: [.image(.init(source: .url(url)))]) + ), + ]) + let archived = try JSONEncoder().encode(transcript) + let decoded = try JSONDecoder().decode(Transcript.self, from: archived) + #expect(decoded == transcript) + _ = try await provider.makeSession(transcript: decoded).respond(to: "Review the image") + try #require(UsageURLProtocol.recordedBodies.count == 1) + try expectToolImage( + in: try #require(UsageURLProtocol.recordedBodies.first), + provider: provider, + texts: [""], + mimeType: nil + ) + } + } + + @Test(arguments: [Provider.anthropic, .gemini], [false, true]) + func missingToolImageFilesStillFail(_ provider: Provider, _ streaming: Bool) async throws { + UsageURLProtocol.reset() + UsageURLProtocol.enqueue( + json: try streaming ? provider.toolStream() : Self.json(provider.response(text: "", tool: true)) + ) + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString).appendingPathExtension("png") + let session = provider.makeSession(tools: [LocalImageWeatherTool(url: url)]) + do { + if streaming { + _ = try await session.streamResponse(to: "Weather?").collect() + } else { + _ = try await session.respond(to: "Weather?") + } + Issue.record("A read failure must not become an image placeholder") + } catch let error as LanguageModelSession.ToolCallError { + #expect((error.underlyingError as NSError).domain == NSCocoaErrorDomain) + #expect((error.underlyingError as NSError).code == NSFileReadNoSuchFileError) + } + #expect(UsageURLProtocol.recordedBodies.count == 1) + } + + @Test(arguments: [Provider.anthropic, .gemini]) + func legacySupportedToolImageReadFailureIsNotAPlaceholder(_ provider: Provider) async throws { + UsageURLProtocol.reset() + let url = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString).appendingPathExtension("png") + let transcript = Transcript(entries: [ + .toolCalls(.init([.init(id: "call_1", toolName: "getWeather", arguments: GeneratedContent("Paris"))])), + .toolOutput(.init(id: "call_1", toolName: "getWeather", segments: [.image(.init(source: .url(url)))])), + ]) + do { + _ = try await provider.makeSession(transcript: transcript).respond(to: "Review the image") + Issue.record("A supported image read failure must remain explicit") + } catch { + #expect((error as NSError).domain == NSCocoaErrorDomain) + #expect((error as NSError).code == NSFileReadNoSuchFileError) + } + #expect(UsageURLProtocol.recordedBodies.isEmpty) + } + + @Test(arguments: [Provider.anthropic, .gemini]) + func unsupportedUserImageMIMEsRemainImages(_ provider: Provider) async throws { + UsageURLProtocol.reset() + UsageURLProtocol.enqueue(json: try Self.json(provider.response())) + let transcript = Transcript(entries: [ + .prompt(.init(segments: [.image(.init(data: testImageData, mimeType: "image/tiff"))])) + ]) + _ = try await provider.makeSession(transcript: transcript).respond(to: "Review the image") + let body = try #require(UsageURLProtocol.recordedBodies.first) + let request = try JSONDecoder().decode([String: JSONValue].self, from: body) + if provider == .gemini { + let images = + request["contents"]?.arrayValue?.flatMap { $0.objectValue?["parts"]?.arrayValue ?? [] } + .compactMap { $0.objectValue?["inlineData"]?.objectValue } ?? [] + #expect(images.first?["mime_type"] == .string("image/tiff")) + } else { + let images = + request["messages"]?.arrayValue?.flatMap { $0.objectValue?["content"]?.arrayValue ?? [] } + .filter { $0.objectValue?["type"] == .string("image") } ?? [] + #expect(images.first?.objectValue?["source"]?.objectValue?["media_type"] == .string("image/tiff")) + } + } + private struct DeleteImageAfterToolOutput: ToolExecutionDelegate { let url: URL + var mimeType = "image/png" func didExecuteToolCall( _ toolCall: Transcript.ToolCall, @@ -550,7 +796,7 @@ import Testing Issue.record("Missing image in Tool output") return } - #expect(image.source == .data(testImageData, mimeType: "image/png")) + #expect(image.source == .data(testImageData, mimeType: mimeType)) do { try FileManager.default.removeItem(at: url) } catch { Issue.record(error) } } }