Preserve attachments in tool outputs

README.md

@@ -382,6 +382,31 @@ pass the model's multimodal projector with `mmprojPath:`.
 > Image inputs are an AnyLanguageModel extension.
 > See [Differences from Foundation Models](#differences-from-foundation-models).
 
+### Prompt attachments
+
+Image attachments use the Foundation Models 27 prompt construction syntax:
+
+```swift
+let prompt = Prompt {
+    "Describe this image"
+    Attachment(imageURL: fileURL)
+}
+```
+
+On Apple platforms, `Attachment(CGImage, orientation:)` retains typed image
+content and orientation. Prompt builders preserve the order of text and images,
+and the OS 26 compatibility transcript continues to store images using its
+existing `Transcript.Segment.image` Codable representation.
+
+`SystemLanguageModel` and `FoundationLanguageModel` bridge prompt attachments
+to Foundation Models on OS 27. Earlier system runtimes reject them explicitly.
+The system bridge requires local file URLs; other provider adapters retain URL
+images in their existing transcript representation.
+
+This first compatibility surface covers prompt `CGImage` and `imageURL`
+attachments. Tool outputs, instruction attachments, labels, `CIImage`, pixel
+buffers, and newer data attachments remain outside this change.
+
 ### Tool Calling
 
 Tool calling is supported by all providers.
@@ -611,8 +636,8 @@ say which API they follow.
   described in that provider's section under [Providers](#providers).
 - `respond(to:images:)`, `Transcript.ImageSegment`, and `Transcript.Segment.image`:
   [image inputs](#image-inputs).
-  Foundation Models 27 adds prompt attachments,
-  and AnyLanguageModel 2.0 will change to match them.
+  Foundation Models 27-style [prompt attachments](#prompt-attachments) are also
+  available for ordered text and image prompt construction.
 - `ToolExecutionDelegate`, `ToolExecutionDecision`, and `toolExecutionDelegate`:
   [observing and controlling tool calls](#tool-calling).
 - `transcriptErrorHandlingPolicy` and `waitForResponseCompletion()`:
@@ -645,7 +670,7 @@ say which API they follow.
 ### Behavior differences
 
 - A session adds the prompt to its transcript before the model responds,
-  as a single text segment.
+  preserving its text and image segment order.
 - A structured response is recorded in the transcript as JSON text,
   not as a structured segment.
 - A session created from a transcript restores `instructions`

Sources/AnyLanguageModel/Attachment.swift

@@ -0,0 +1,107 @@
+import Foundation
+
+#if canImport(CoreGraphics) && canImport(ImageIO)
+    import CoreGraphics
+    import ImageIO
+#endif
+
+/// A typed prompt attachment, following Foundation Models 27's image surface.
+///
+/// AnyLanguageModel lowers image attachments into its existing compatibility
+/// transcript image segments on deployment targets before OS 27.
+public struct Attachment<Content> {
+    let content: Content
+}
+
+/// Image content retained until a prompt is lowered into the compatibility transcript.
+public struct ImageAttachmentContent: Sendable, Equatable {
+    enum Source: Sendable {
+        case url(URL, orientation: UInt32?)
+        #if canImport(CoreGraphics) && canImport(ImageIO)
+            case image(CGImage, orientation: CGImagePropertyOrientation?)
+        #endif
+    }
+
+    let source: Source
+
+    public static func == (lhs: Self, rhs: Self) -> Bool {
+        switch (lhs.source, rhs.source) {
+        case (.url(let a, let ao), .url(let b, let bo)):
+            return a == b && ao == bo
+        #if canImport(CoreGraphics) && canImport(ImageIO)
+            case (.image(let a, let ao), .image(let b, let bo)):
+                return a === b && ao == bo
+        #endif
+        default:
+            return false
+        }
+    }
+
+    func makeSegment(id: String) throws -> Transcript.ImageSegment {
+        switch source {
+        case .url(let url, let orientation):
+            #if canImport(CoreGraphics) && canImport(ImageIO)
+                if let orientation {
+                    guard let source = CGImageSourceCreateWithURL(url as CFURL, nil),
+                        let image = CGImageSourceCreateImageAtIndex(source, 0, nil)
+                    else {
+                        throw Transcript.ImageEncodingError.imageConversionFailed
+                    }
+                    return try Self(
+                        source: .image(
+                            image,
+                            orientation: CGImagePropertyOrientation(rawValue: orientation)
+                        )
+                    ).makeSegment(id: id)
+                }
+            #endif
+            return .init(id: id, url: url)
+        #if canImport(CoreGraphics) && canImport(ImageIO)
+            case .image(let image, let orientation):
+                let bytes = NSMutableData()
+                guard
+                    let destination = CGImageDestinationCreateWithData(
+                        bytes,
+                        "public.png" as CFString,
+                        1,
+                        nil
+                    )
+                else {
+                    throw Transcript.ImageEncodingError.imageConversionFailed
+                }
+                var properties: [CFString: Any] = [:]
+                if let orientation {
+                    properties[kCGImagePropertyOrientation] = orientation.rawValue
+                }
+                CGImageDestinationAddImage(destination, image, properties as CFDictionary)
+                guard CGImageDestinationFinalize(destination) else {
+                    throw Transcript.ImageEncodingError.imageConversionFailed
+                }
+                return .init(id: id, data: bytes as Data, mimeType: "image/png")
+        #endif
+        }
+    }
+}
+
+extension Attachment where Content == ImageAttachmentContent {
+    #if canImport(CoreGraphics) && canImport(ImageIO)
+        public init(_ image: CGImage, orientation: CGImagePropertyOrientation? = nil) {
+            content = .init(source: .image(image, orientation: orientation))
+        }
+
+        public init(imageURL: URL, orientation: CGImagePropertyOrientation? = nil) {
+            // Preserve the URL itself; its bytes are not read merely by constructing a prompt.
+            content = .init(source: .url(imageURL, orientation: orientation?.rawValue))
+        }
+    #else
+        public init(imageURL: URL) {
+            content = .init(source: .url(imageURL, orientation: nil))
+        }
+    #endif
+}
+
+extension Attachment: PromptRepresentable where Content == ImageAttachmentContent {
+    public var promptRepresentation: Prompt {
+        Prompt(components: [.image(id: UUID().uuidString, content: content)])
+    }
+}

Sources/AnyLanguageModel/LanguageModelSession.swift

@@ -728,7 +728,7 @@ public final class LanguageModelSession: @unchecked Sendable {
             let promptEntry = Transcript.Entry.prompt(
                 Transcript.Prompt(
                     id: promptID,
-                    segments: [.text(.init(content: prompt.description))],
+                    segments: try prompt.makeTranscriptSegments(),
                     options: options,
                     responseFormat: responseFormat
                 )
@@ -776,9 +776,15 @@ public final class LanguageModelSession: @unchecked Sendable {
         options: GenerationOptions = GenerationOptions()
     ) -> sending ResponseStream<Content> where Content: Generable {
         // Add prompt to transcript
+        let segments: [Transcript.Segment]
+        do {
+            segments = try prompt.makeTranscriptSegments()
+        } catch {
+            return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) })
+        }
         let promptEntry = Transcript.Entry.prompt(
             Transcript.Prompt(
-                segments: [.text(.init(content: prompt.description))],
+                segments: segments,
                 options: options,
                 responseFormat: type == String.self ? nil : .init(type: type)
             )
@@ -924,9 +930,15 @@ extension LanguageModelSession {
         includeSchemaInPrompt: Bool = true,
         options: GenerationOptions = GenerationOptions()
     ) -> sending ResponseStream<GeneratedContent> {
+        let segments: [Transcript.Segment]
+        do {
+            segments = try prompt.makeTranscriptSegments()
+        } catch {
+            return ResponseStream(stream: AsyncThrowingStream { $0.finish(throwing: error) })
+        }
         let promptEntry = Transcript.Entry.prompt(
             Transcript.Prompt(
-                segments: [.text(.init(content: prompt.description))],
+                segments: segments,
                 options: options,
                 responseFormat: .init(schema: schema)
             )

Sources/AnyLanguageModel/Models/FoundationLanguageModel.swift

@@ -159,7 +159,7 @@
         ) async throws -> LanguageModelSession.Response<Content> where Content: Generable {
             return try await fmRespond(
                 makeSession: { try await self.makeSession(for: session, prompt: prompt) },
-                fmPrompt: prompt.toFoundationModels(),
+                fmPrompt: try prompt.toFoundationModels(),
                 fmOptions: options.toFoundationModels(),
                 type: type,
                 schema: schema,
@@ -209,9 +209,15 @@
             includeSchemaInPrompt: Bool,
             options: GenerationOptions
         ) -> sending LanguageModelSession.ResponseStream<Content> where Content: Generable {
+            let fmPrompt: FoundationModels.Prompt
+            do {
+                fmPrompt = try prompt.toFoundationModels()
+            } catch {
+                return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) })
+            }
             return fmStreamResponse(
                 makeSession: { try await self.makeSession(for: session, prompt: prompt) },
-                fmPrompt: prompt.toFoundationModels(),
+                fmPrompt: fmPrompt,
                 fmOptions: options.toFoundationModels(),
                 type: type,
                 schema: schema,

Sources/AnyLanguageModel/Models/SystemLanguageModel.swift

@@ -194,7 +194,7 @@
             includeSchemaInPrompt: Bool,
             options: GenerationOptions
         ) async throws -> LanguageModelSession.Response<Content> where Content: Generable {
-            let fmPrompt = prompt.toFoundationModels()
+            let fmPrompt = try prompt.toFoundationModels()
             let fmOptions = options.toFoundationModels()
 
             return try await fmRespond(
@@ -249,7 +249,12 @@
             includeSchemaInPrompt: Bool,
             options: GenerationOptions
         ) -> sending LanguageModelSession.ResponseStream<Content> where Content: Generable {
-            let fmPrompt = prompt.toFoundationModels()
+            let fmPrompt: FoundationModels.Prompt
+            do {
+                fmPrompt = try prompt.toFoundationModels()
+            } catch {
+                return .init(stream: AsyncThrowingStream { $0.finish(throwing: error) })
+            }
             let fmOptions = options.toFoundationModels()
 
             return fmStreamResponse(
@@ -336,11 +341,9 @@
         guard let lastEntry = transcript.last, case .prompt(let lastPrompt) = lastEntry else {
             return transcript
         }
-        let lastText = lastPrompt.segments.compactMap { segment -> String? in
-            if case .text(let textSegment) = segment { return textSegment.content }
-            return nil
-        }.joined()
-        guard lastText == prompt.description else {
+        guard let promptSegments = try? prompt.makeTranscriptSegments(),
+            lastPrompt.segments == promptSegments
+        else {
             return transcript
         }
         return Transcript(entries: transcript.dropLast())
@@ -430,8 +433,42 @@
 
     @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *)
     extension Prompt {
-        func toFoundationModels() -> FoundationModels.Prompt {
-            FoundationModels.Prompt(self.description)
+        func toFoundationModels() throws -> FoundationModels.Prompt {
+            let prompts: [FoundationModels.Prompt] = try components.map { component in
+                switch component {
+                case .text(let text):
+                    return FoundationModels.Prompt(text.content)
+                case .image(_, let image):
+                    #if compiler(>=6.4) && !os(tvOS) && !os(watchOS)
+                        if #available(macOS 27, iOS 27, visionOS 27, *) {
+                            switch image.source {
+                            case .url(let url, let orientation):
+                                guard url.isFileURL else {
+                                    throw PromptBridgeError.nonFileImageURL
+                                }
+                                return FoundationModels.Prompt(
+                                    FoundationModels.Attachment(
+                                        imageURL: url,
+                                        orientation: orientation.flatMap(
+                                            CGImagePropertyOrientation.init(rawValue:)
+                                        )
+                                    )
+                                )
+                            case .image(let value, let orientation):
+                                return FoundationModels.Prompt(
+                                    FoundationModels.Attachment(value, orientation: orientation)
+                                )
+                            }
+                        }
+                    #endif
+                    throw PromptBridgeError.attachmentsUnavailable
+                }
+            }
+            return FoundationModels.Prompt {
+                for prompt in prompts {
+                    prompt
+                }
+            }
         }
     }
 
@@ -517,11 +554,16 @@
         }
     }
 
+    enum PromptBridgeError: Error {
+        case attachmentsUnavailable
+        case nonFileImageURL
+    }
+
     /// A type-erased wrapper that bridges any `Tool` to `FoundationModels.Tool`.
     @available(macOS 26.0, iOS 26.0, watchOS 27.0, tvOS 26.0, visionOS 26.0, *)
     private struct AnyToolWrapper: FoundationModels.Tool {
         typealias Arguments = FoundationModels.GeneratedContent
-        typealias Output = String
+        typealias Output = FoundationModels.Prompt
 
         let name: String
         let description: String
@@ -540,7 +582,7 @@
 
         func call(arguments: FoundationModels.GeneratedContent) async throws -> Output {
             let output = try await wrappedTool.callFunction(arguments: arguments)
-            return output.promptRepresentation.description
+            return try output.promptRepresentation.toFoundationModels()
         }
     }
 
@@ -924,7 +966,15 @@
                     else {
                         return nil
                     }
-                    self.init(cgImage)
+                    let properties =
+                        CGImageSourceCopyPropertiesAtIndex(
+                            imageSource,
+                            0,
+                            nil
+                        ) as? [CFString: Any]
+                    let orientation = (properties?[kCGImagePropertyOrientation] as? NSNumber)
+                        .flatMap { CGImagePropertyOrientation(rawValue: $0.uint32Value) }
+                    self.init(cgImage, orientation: orientation)
                 }
             }
         }