Add support for image inputs

Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift

@@ -100,8 +100,9 @@ public struct AnthropicLanguageModel: LanguageModel {
         let url = baseURL.appendingPathComponent("v1/messages")
         let headers = buildHeaders()
 
+        let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description)
         let messages = [
-            AnthropicMessage(role: .user, content: [.text(.init(text: prompt.description))])
+            AnthropicMessage(role: .user, content: convertSegmentsToAnthropicContent(userSegments))
         ]
 
         // Convert available tools to Anthropic format
@@ -170,8 +171,9 @@ public struct AnthropicLanguageModel: LanguageModel {
             fatalError("AnthropicLanguageModel only supports generating String content")
         }
 
+        let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description)
         let messages = [
-            AnthropicMessage(role: .user, content: [.text(.init(text: prompt.description))])
+            AnthropicMessage(role: .user, content: convertSegmentsToAnthropicContent(userSegments))
         ]
 
         let url = baseURL.appendingPathComponent("v1/messages")
@@ -383,18 +385,21 @@ private struct AnthropicMessage: Codable, Sendable {
 
 private enum AnthropicContent: Codable, Sendable {
     case text(AnthropicText)
+    case image(AnthropicImage)
     case toolUse(AnthropicToolUse)
 
     enum CodingKeys: String, CodingKey { case type }
 
-    enum ContentType: String, Codable { case text = "text", toolUse = "tool_use" }
+    enum ContentType: String, Codable { case text = "text", image = "image", toolUse = "tool_use" }
 
     init(from decoder: any Decoder) throws {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         let type = try container.decode(ContentType.self, forKey: .type)
         switch type {
         case .text:
             self = .text(try AnthropicText(from: decoder))
+        case .image:
+            self = .image(try AnthropicImage(from: decoder))
         case .toolUse:
             self = .toolUse(try AnthropicToolUse(from: decoder))
         }
@@ -403,6 +408,7 @@ private enum AnthropicContent: Codable, Sendable {
     func encode(to encoder: any Encoder) throws {
         switch self {
         case .text(let t): try t.encode(to: encoder)
+        case .image(let i): try i.encode(to: encoder)
         case .toolUse(let u): try u.encode(to: encoder)
         }
     }
@@ -418,6 +424,77 @@ private struct AnthropicText: Codable, Sendable {
     }
 }
 
+private struct AnthropicImage: Codable, Sendable {
+    struct Source: Codable, Sendable {
+        let type: String
+        let mediaType: String?
+        let data: String?
+        let url: String?
+
+        enum CodingKeys: String, CodingKey {
+            case type
+            case mediaType = "media_type"
+            case data
+            case url
+        }
+    }
+
+    let type: String
+    let source: Source
+
+    init(base64Data: String, mimeType: String) {
+        self.type = "image"
+        self.source = Source(type: "base64", mediaType: mimeType, data: base64Data, url: nil)
+    }
+
+    init(url: String) {
+        self.type = "image"
+        self.source = Source(type: "url", mediaType: nil, data: nil, url: url)
+    }
+}
+
+private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) -> [AnthropicContent] {
+    var blocks: [AnthropicContent] = []
+    blocks.reserveCapacity(segments.count)
+    for segment in segments {
+        switch segment {
+        case .text(let t):
+            blocks.append(.text(AnthropicText(text: t.content)))
+        case .structure(let s):
+            blocks.append(.text(AnthropicText(text: s.content.jsonString)))
+        case .image(let img):
+            switch img.source {
+            case .url(let url):
+                blocks.append(.image(AnthropicImage(url: url.absoluteString)))
+            case .data(let data, let mimeType):
+                blocks.append(.image(AnthropicImage(base64Data: data.base64EncodedString(), mimeType: mimeType)))
+            }
+        }
+    }
+    return blocks
+}
+
+private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] {
+    for entry in session.transcript.reversed() {
+        if case .prompt(let p) = entry {
+            // Skip prompts that are effectively empty (single empty text block)
+            let hasMeaningfulContent = p.segments.contains { segment in
+                switch segment {
+                case .text(let t):
+                    return !t.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+                case .structure:
+                    return true
+                case .image:
+                    return true
+                }
+            }
+            if hasMeaningfulContent { return p.segments }
+            // Otherwise continue searching older entries
+        }
+    }
+    return [.text(.init(content: fallbackText))]
+}
+
 private struct AnthropicToolUse: Codable, Sendable {
     let type: String
     let id: String

Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift

@@ -210,6 +210,8 @@
 
         /// The model file was found but is corrupted, incompatible, or otherwise invalid.
         case modelInvalid(URL, underlyingError: Error)
+        /// Image segments are not supported in CoreMLLanguageModel
+        case unsupportedFeature
 
         public var errorDescription: String? {
             switch self {
@@ -221,6 +223,8 @@
             case .modelInvalid(let url, let underlyingError):
                 return
                     "Core ML model at \(url.path) is invalid or corrupted: \(underlyingError.localizedDescription). Please verify the model file is valid and compatible with the current Core ML version."
+            case .unsupportedFeature:
+                return "This CoreMLLanguageModel does not support image segments"
             }
         }
     }
@@ -245,7 +249,7 @@
                 config.topK = k
             case .nucleus(let p, _):
                 config.doSample = true
-                config.topP = p
+                config.topP = Float(p)
             }
         }
 

Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift

@@ -97,8 +97,9 @@ public struct GeminiLanguageModel: LanguageModel {
             .appendingPathComponent("models/\(model):generateContent")
         let headers = buildHeaders()
 
+        let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description)
         var contents = [
-            GeminiContent(role: .user, parts: [.text(GeminiTextPart(text: prompt.description))])
+            GeminiContent(role: .user, parts: convertSegmentsToGeminiParts(userSegments))
         ]
 
         let geminiTools = try buildTools(from: session.tools)
@@ -195,8 +196,9 @@ public struct GeminiLanguageModel: LanguageModel {
             fatalError("GeminiLanguageModel only supports generating String content")
         }
 
+        let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description)
         let contents = [
-            GeminiContent(role: .user, parts: [.text(GeminiTextPart(text: prompt.description))])
+            GeminiContent(role: .user, parts: convertSegmentsToGeminiParts(userSegments))
         ]
 
         var streamURL =
@@ -442,6 +444,9 @@ private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String:
             if let jsonString = String(data: data, encoding: .utf8) {
                 result["result"] = .string(jsonString)
             }
+        case .image:
+            // Ignore images in tool outputs for Gemini conversion
+            break
         }
     }
 
@@ -511,12 +516,16 @@ private enum GeminiPart: Codable, Sendable {
     case text(GeminiTextPart)
     case functionCall(GeminiFunctionCall)
     case functionResponse(GeminiFunctionResponse)
+    case inlineData(GeminiInlineData)
+    case fileData(GeminiFileData)
 
     enum CodingKeys: String, CodingKey {
         case text
         case functionCall
         case functionResponse
         case thoughtSignature
+        case inlineData
+        case fileData
     }
 
     init(from decoder: any Decoder) throws {
@@ -530,6 +539,10 @@ private enum GeminiPart: Codable, Sendable {
             self = .functionCall(try container.decode(GeminiFunctionCall.self, forKey: .functionCall))
         } else if container.contains(.functionResponse) {
             self = .functionResponse(try container.decode(GeminiFunctionResponse.self, forKey: .functionResponse))
+        } else if container.contains(.inlineData) {
+            self = .inlineData(try container.decode(GeminiInlineData.self, forKey: .inlineData))
+        } else if container.contains(.fileData) {
+            self = .fileData(try container.decode(GeminiFileData.self, forKey: .fileData))
         } else {
             throw DecodingError.dataCorrupted(
                 DecodingError.Context(
@@ -549,6 +562,10 @@ private enum GeminiPart: Codable, Sendable {
             try container.encode(call, forKey: .functionCall)
         case .functionResponse(let response):
             try container.encode(response, forKey: .functionResponse)
+        case .inlineData(let data):
+            try container.encode(data, forKey: .inlineData)
+        case .fileData(let data):
+            try container.encode(data, forKey: .fileData)
         }
     }
 }
@@ -557,6 +574,54 @@ private struct GeminiTextPart: Codable, Sendable {
     let text: String
 }
 
+private struct GeminiInlineData: Codable, Sendable {
+    let mimeType: String
+    let data: String
+
+    enum CodingKeys: String, CodingKey {
+        case mimeType = "mime_type"
+        case data
+    }
+}
+
+private struct GeminiFileData: Codable, Sendable {
+    let fileURI: String
+
+    enum CodingKeys: String, CodingKey {
+        case fileURI = "file_uri"
+    }
+}
+
+private func convertSegmentsToGeminiParts(_ segments: [Transcript.Segment]) -> [GeminiPart] {
+    var parts: [GeminiPart] = []
+    parts.reserveCapacity(segments.count)
+    for segment in segments {
+        switch segment {
+        case .text(let t):
+            parts.append(.text(GeminiTextPart(text: t.content)))
+        case .structure(let s):
+            parts.append(.text(GeminiTextPart(text: s.content.jsonString)))
+        case .image(let img):
+            switch img.source {
+            case .data(let data, let mime):
+                parts.append(.inlineData(GeminiInlineData(mimeType: mime, data: data.base64EncodedString())))
+            case .url(let url):
+                parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString)))
+            }
+        }
+    }
+    return parts
+}
+
+private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] {
+    for entry in session.transcript.reversed() {
+        if case .prompt(let p) = entry {
+            return p.segments
+        }
+    }
+    return [.text(.init(content: fallbackText))]
+}
+
 private struct GeminiFunctionCall: Codable, Sendable {
     let name: String
     let args: [String: JSONValue]?

Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift

@@ -601,6 +601,7 @@ import Foundation
         case decodingFailed
         case invalidModelPath
         case insufficientMemory
+        case unsupportedFeature
 
         public var errorDescription: String? {
             switch self {
@@ -618,6 +619,8 @@ import Foundation
                 return "Invalid model file path"
             case .insufficientMemory:
                 return "Insufficient memory for operation"
+            case .unsupportedFeature:
+                return "This LlamaLanguageModel does not support image segments"
             }
         }
     }

Sources/AnyLanguageModel/Models/MLXLanguageModel.swift

@@ -148,6 +148,19 @@ import Foundation
         }
     }
 
+    /// Errors that can occur when working with MLX language models.
+    public enum MLXLanguageModelError: LocalizedError {
+        /// Image segments are not supported in MLXLanguageModel
+        case unsupportedFeature
+
+        public var errorDescription: String? {
+            switch self {
+            case .unsupportedFeature:
+                return "This MLXLanguageModel does not support image segments"
+            }
+        }
+    }
+
     // MARK: - Options Mapping
 
     private func toGenerateParameters(_ options: GenerationOptions) -> MLXLMCommon.GenerateParameters {
@@ -267,6 +280,9 @@ import Foundation
             case .structure(let structuredSegment):
                 // structured content already has jsonString property
                 textParts.append(structuredSegment.content.jsonString)
+            case .image:
+                // Image segments are not supported in MLX tool output
+                break
             }
         }
         return textParts.joined(separator: "\n")