|
@@ -438,6 +438,26 @@ export namespace MessageV2 {
|
|
|
export function toModelMessages(input: WithParts[], model: Provider.Model): ModelMessage[] {
|
|
export function toModelMessages(input: WithParts[], model: Provider.Model): ModelMessage[] {
|
|
|
const result: UIMessage[] = []
|
|
const result: UIMessage[] = []
|
|
|
const toolNames = new Set<string>()
|
|
const toolNames = new Set<string>()
|
|
|
|
|
+ // Track media from tool results that need to be injected as user messages
|
|
|
|
|
+ // for providers that don't support media in tool results.
|
|
|
|
|
+ //
|
|
|
|
|
+ // OpenAI-compatible APIs only support string content in tool results, so we need
|
|
|
|
|
+ // to extract media and inject as user messages. Other SDKs (anthropic, google,
|
|
|
|
|
+ // bedrock) handle type: "content" with media parts natively.
|
|
|
|
|
+ //
|
|
|
|
|
+ // Only apply this workaround if the model actually supports image input -
|
|
|
|
|
+ // otherwise there's no point extracting images.
|
|
|
|
|
+ const supportsMediaInToolResults = (() => {
|
|
|
|
|
+ if (model.api.npm === "@ai-sdk/anthropic") return true
|
|
|
|
|
+ if (model.api.npm === "@ai-sdk/openai") return true
|
|
|
|
|
+ if (model.api.npm === "@ai-sdk/amazon-bedrock") return true
|
|
|
|
|
+ if (model.api.npm === "@ai-sdk/google-vertex/anthropic") return true
|
|
|
|
|
+ if (model.api.npm === "@ai-sdk/google") {
|
|
|
|
|
+ const id = model.api.id.toLowerCase()
|
|
|
|
|
+ return id.includes("gemini-3") && !id.includes("gemini-2")
|
|
|
|
|
+ }
|
|
|
|
|
+ return false
|
|
|
|
|
+ })()
|
|
|
|
|
|
|
|
const toModelOutput = (output: unknown) => {
|
|
const toModelOutput = (output: unknown) => {
|
|
|
if (typeof output === "string") {
|
|
if (typeof output === "string") {
|
|
@@ -514,6 +534,7 @@ export namespace MessageV2 {
|
|
|
|
|
|
|
|
if (msg.info.role === "assistant") {
|
|
if (msg.info.role === "assistant") {
|
|
|
const differentModel = `${model.providerID}/${model.id}` !== `${msg.info.providerID}/${msg.info.modelID}`
|
|
const differentModel = `${model.providerID}/${model.id}` !== `${msg.info.providerID}/${msg.info.modelID}`
|
|
|
|
|
+ const media: Array<{ mime: string; url: string }> = []
|
|
|
|
|
|
|
|
if (
|
|
if (
|
|
|
msg.info.error &&
|
|
msg.info.error &&
|
|
@@ -545,11 +566,23 @@ export namespace MessageV2 {
|
|
|
if (part.state.status === "completed") {
|
|
if (part.state.status === "completed") {
|
|
|
const outputText = part.state.time.compacted ? "[Old tool result content cleared]" : part.state.output
|
|
const outputText = part.state.time.compacted ? "[Old tool result content cleared]" : part.state.output
|
|
|
const attachments = part.state.time.compacted ? [] : (part.state.attachments ?? [])
|
|
const attachments = part.state.time.compacted ? [] : (part.state.attachments ?? [])
|
|
|
|
|
+
|
|
|
|
|
+ // For providers that don't support media in tool results, extract media files
|
|
|
|
|
+ // (images, PDFs) to be sent as a separate user message
|
|
|
|
|
+ const isMediaAttachment = (a: { mime: string }) =>
|
|
|
|
|
+ a.mime.startsWith("image/") || a.mime === "application/pdf"
|
|
|
|
|
+ const mediaAttachments = attachments.filter(isMediaAttachment)
|
|
|
|
|
+ const nonMediaAttachments = attachments.filter((a) => !isMediaAttachment(a))
|
|
|
|
|
+ if (!supportsMediaInToolResults && mediaAttachments.length > 0) {
|
|
|
|
|
+ media.push(...mediaAttachments)
|
|
|
|
|
+ }
|
|
|
|
|
+ const finalAttachments = supportsMediaInToolResults ? attachments : nonMediaAttachments
|
|
|
|
|
+
|
|
|
const output =
|
|
const output =
|
|
|
- attachments.length > 0
|
|
|
|
|
|
|
+ finalAttachments.length > 0
|
|
|
? {
|
|
? {
|
|
|
text: outputText,
|
|
text: outputText,
|
|
|
- attachments,
|
|
|
|
|
|
|
+ attachments: finalAttachments,
|
|
|
}
|
|
}
|
|
|
: outputText
|
|
: outputText
|
|
|
|
|
|
|
@@ -593,6 +626,25 @@ export namespace MessageV2 {
|
|
|
}
|
|
}
|
|
|
if (assistantMessage.parts.length > 0) {
|
|
if (assistantMessage.parts.length > 0) {
|
|
|
result.push(assistantMessage)
|
|
result.push(assistantMessage)
|
|
|
|
|
+ // Inject pending media as a user message for providers that don't support
|
|
|
|
|
+ // media (images, PDFs) in tool results
|
|
|
|
|
+ if (media.length > 0) {
|
|
|
|
|
+ result.push({
|
|
|
|
|
+ id: Identifier.ascending("message"),
|
|
|
|
|
+ role: "user",
|
|
|
|
|
+ parts: [
|
|
|
|
|
+ {
|
|
|
|
|
+ type: "text" as const,
|
|
|
|
|
+ text: "Attached image(s) from tool result:",
|
|
|
|
|
+ },
|
|
|
|
|
+ ...media.map((attachment) => ({
|
|
|
|
|
+ type: "file" as const,
|
|
|
|
|
+ url: attachment.url,
|
|
|
|
|
+ mediaType: attachment.mime,
|
|
|
|
|
+ })),
|
|
|
|
|
+ ],
|
|
|
|
|
+ })
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|