copilot-chat-model.test.ts 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557
  1. import { OpenAICompatibleChatLanguageModel } from "@/provider/sdk/copilot/chat/openai-compatible-chat-language-model"
  2. import { describe, test, expect, mock } from "bun:test"
  3. import type { LanguageModelV2Prompt } from "@ai-sdk/provider"
  4. async function convertReadableStreamToArray<T>(stream: ReadableStream<T>): Promise<T[]> {
  5. const reader = stream.getReader()
  6. const result: T[] = []
  7. while (true) {
  8. const { done, value } = await reader.read()
  9. if (done) break
  10. result.push(value)
  11. }
  12. return result
  13. }
  14. const TEST_PROMPT: LanguageModelV2Prompt = [{ role: "user", content: [{ type: "text", text: "Hello" }] }]
  15. // Fixtures from copilot_test.exs
  16. const FIXTURES = {
  17. basicText: [
  18. `data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gemini-2.0-flash-001","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"},"finish_reason":null}]}`,
  19. `data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gemini-2.0-flash-001","choices":[{"index":0,"delta":{"content":" world"},"finish_reason":null}]}`,
  20. `data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gemini-2.0-flash-001","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":"stop"}]}`,
  21. `data: [DONE]`,
  22. ],
  23. reasoningWithToolCalls: [
  24. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Understanding Dayzee's Purpose**\\n\\nI'm starting to get a better handle on \`dayzee\`.\\n\\n"}}],"created":1764940861,"id":"OdwyabKMI9yel7oPlbzgwQM","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  25. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Assessing Dayzee's Functionality**\\n\\nI've reviewed the files.\\n\\n"}}],"created":1764940862,"id":"OdwyabKMI9yel7oPlbzgwQM","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  26. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\\"filePath\\":\\"/README.md\\"}","name":"read_file"},"id":"call_abc123","index":0,"type":"function"}],"reasoning_opaque":"4CUQ6696CwSXOdQ5rtvDimqA91tBzfmga4ieRbmZ5P67T2NLW3"}}],"created":1764940862,"id":"OdwyabKMI9yel7oPlbzgwQM","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  27. `data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{"content":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\\"filePath\\":\\"/mix.exs\\"}","name":"read_file"},"id":"call_def456","index":1,"type":"function"}]}}],"created":1764940862,"id":"OdwyabKMI9yel7oPlbzgwQM","usage":{"completion_tokens":53,"prompt_tokens":19581,"prompt_tokens_details":{"cached_tokens":17068},"total_tokens":19768,"reasoning_tokens":134},"model":"gemini-3-pro-preview"}`,
  28. `data: [DONE]`,
  29. ],
  30. reasoningWithOpaqueAtEnd: [
  31. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Analyzing the Inquiry's Nature**\\n\\nI'm currently parsing the user's question.\\n\\n"}}],"created":1765201729,"id":"Ptc2afqsCIHqlOoP653UiAI","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  32. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Reconciling User's Input**\\n\\nI'm grappling with the context.\\n\\n"}}],"created":1765201730,"id":"Ptc2afqsCIHqlOoP653UiAI","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  33. `data: {"choices":[{"index":0,"delta":{"content":"I am Tidewave, a highly skilled AI coding agent.\\n\\n","role":"assistant"}}],"created":1765201730,"id":"Ptc2afqsCIHqlOoP653UiAI","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  34. `data: {"choices":[{"finish_reason":"stop","index":0,"delta":{"content":"How can I help you?","role":"assistant","reasoning_opaque":"/PMlTqxqSJZnUBDHgnnJKLVI4eZQ"}}],"created":1765201730,"id":"Ptc2afqsCIHqlOoP653UiAI","usage":{"completion_tokens":59,"prompt_tokens":5778,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":5932,"reasoning_tokens":95},"model":"gemini-3-pro-preview"}`,
  35. `data: [DONE]`,
  36. ],
  37. // Case where reasoning_opaque and content come in the SAME chunk
  38. reasoningWithOpaqueAndContentSameChunk: [
  39. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Understanding the Query's Nature**\\n\\nI'm currently grappling with the user's philosophical query.\\n\\n"}}],"created":1766062103,"id":"FPhDacixL9zrlOoPqLSuyQ4","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-2.5-pro"}`,
  40. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Framing the Response's Core**\\n\\nNow, I'm structuring my response.\\n\\n"}}],"created":1766062103,"id":"FPhDacixL9zrlOoPqLSuyQ4","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-2.5-pro"}`,
  41. `data: {"choices":[{"index":0,"delta":{"content":"Of course. I'm thinking right now.","role":"assistant","reasoning_opaque":"ExXaGwW7jBo39OXRe9EPoFGN1rOtLJBx"}}],"created":1766062103,"id":"FPhDacixL9zrlOoPqLSuyQ4","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-2.5-pro"}`,
  42. `data: {"choices":[{"finish_reason":"stop","index":0,"delta":{"content":" What's on your mind?","role":"assistant"}}],"created":1766062103,"id":"FPhDacixL9zrlOoPqLSuyQ4","usage":{"completion_tokens":78,"prompt_tokens":3767,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":3915,"reasoning_tokens":70},"model":"gemini-2.5-pro"}`,
  43. `data: [DONE]`,
  44. ],
  45. // Case where reasoning_opaque and content come in same chunk, followed by tool calls
  46. reasoningWithOpaqueContentAndToolCalls: [
  47. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Analyzing the Structure**\\n\\nI'm currently trying to get a handle on the project's layout. My initial focus is on the file structure itself, specifically the directory organization. I'm hoping this will illuminate how different components interact. I'll need to identify the key modules and their dependencies.\\n\\n\\n"}}],"created":1766066995,"id":"MQtEafqbFYTZsbwPwuCVoAg","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-2.5-pro"}`,
  48. `data: {"choices":[{"index":0,"delta":{"content":"Okay, I need to check out the project's file structure.","role":"assistant","reasoning_opaque":"WHOd3dYFnxEBOsKUXjbX6c2rJa0fS214FHbsj+A3Q+i63SFo7H/92RsownAzyo0h2qEy3cOcrvAatsMx51eCKiMSqt4dYWZhd5YVSgF0CehkpDbWBP/SoRqLU1dhCmUJV/6b5uYFBOzKLBGNadyhI7T1gWFlXntwc6SNjH6DujnFPeVr+L8DdOoUJGJrw2aOfm9NtkXA6wZh9t7dt+831yIIImjD9MHczuXoXj8K7tyLpIJ9KlVXMhnO4IKSYNdKRtoHlGTmudAp5MgH/vLWb6oSsL+ZJl/OdF3WBOeanGhYNoByCRDSvR7anAR/9m5zf9yUax+u/nFg+gzmhFacnzZGtSmcvJ4/4HWKNtUkRASTKeN94DXB8j1ptB/i6ldaMAz2ZyU+sbjPWI8aI4fKJ2MuO01u3uE87xVwpWiM+0rahIzJsllI5edwOaOFtF4tnlCTQafbxHwCZR62uON2E+IjGzW80MzyfYrbLBJKS5zTeHCgPYQSNaKzPfpzkQvdwo3JUnJYcEHgGeKzkq5sbvS5qitCYI7Xue0V98S6/KnUSPnDQBjNnas2i6BqJV2vuCEU/Y3ucrlKVbuRIFCZXCyLzrsGeRLRKlrf5S/HDAQ04IOPQVQhBPvhX0nDjhZB"}}],"created":1766066995,"id":"MQtEafqbFYTZsbwPwuCVoAg","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-2.5-pro"}`,
  49. `data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{"content":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{}","name":"list_project_files"},"id":"call_MHxqRDd5WVo3NU8wUXRaMmc0MFE","index":0,"type":"function"}]}}],"created":1766066995,"id":"MQtEafqbFYTZsbwPwuCVoAg","usage":{"completion_tokens":19,"prompt_tokens":3767,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":3797,"reasoning_tokens":11},"model":"gemini-2.5-pro"}`,
  50. `data: [DONE]`,
  51. ],
  52. // Case where reasoning goes directly to tool_calls with NO content
  53. // reasoning_opaque and tool_calls come in the same chunk
  54. reasoningDirectlyToToolCalls: [
  55. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Executing and Analyzing HTML**\\n\\nI've successfully captured the HTML snapshot using the \`browser_eval\` tool, giving me a solid understanding of the page structure. Now, I'm shifting focus to Elixir code execution with \`project_eval\` to assess my ability to work within the project's environment.\\n\\n\\n"}}],"created":1766068643,"id":"oBFEaafzD9DVlOoPkY3l4Qs","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  56. `data: {"choices":[{"index":0,"delta":{"content":null,"role":"assistant","reasoning_text":"**Testing Project Contexts**\\n\\nI've got the HTML body snapshot from \`browser_eval\`, which is a helpful reference. Next, I'm testing my ability to run Elixir code in the project with \`project_eval\`. I'm starting with a simple sum: \`1 + 1\`. This will confirm I'm set up to interact with the project's codebase.\\n\\n\\n"}}],"created":1766068644,"id":"oBFEaafzD9DVlOoPkY3l4Qs","usage":{"completion_tokens":0,"prompt_tokens":0,"prompt_tokens_details":{"cached_tokens":0},"total_tokens":0,"reasoning_tokens":0},"model":"gemini-3-pro-preview"}`,
  57. `data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{"content":null,"role":"assistant","tool_calls":[{"function":{"arguments":"{\\"code\\":\\"1 + 1\\"}","name":"project_eval"},"id":"call_MHw3RDhmT1J5Z3B6WlhpVjlveTc","index":0,"type":"function"}],"reasoning_opaque":"ytGNWFf2doK38peANDvm7whkLPKrd+Fv6/k34zEPBF6Qwitj4bTZT0FBXleydLb6"}}],"created":1766068644,"id":"oBFEaafzD9DVlOoPkY3l4Qs","usage":{"completion_tokens":12,"prompt_tokens":8677,"prompt_tokens_details":{"cached_tokens":3692},"total_tokens":8768,"reasoning_tokens":79},"model":"gemini-3-pro-preview"}`,
  58. `data: [DONE]`,
  59. ],
  60. }
  61. function createMockFetch(chunks: string[]) {
  62. return mock(async () => {
  63. const body = new ReadableStream({
  64. start(controller) {
  65. for (const chunk of chunks) {
  66. controller.enqueue(new TextEncoder().encode(chunk + "\n\n"))
  67. }
  68. controller.close()
  69. },
  70. })
  71. return new Response(body, {
  72. status: 200,
  73. headers: { "Content-Type": "text/event-stream" },
  74. })
  75. })
  76. }
  77. function createModel(fetchFn: ReturnType<typeof mock>) {
  78. return new OpenAICompatibleChatLanguageModel("test-model", {
  79. provider: "copilot.chat",
  80. url: () => "https://api.test.com/chat/completions",
  81. headers: () => ({ Authorization: "Bearer test-token" }),
  82. fetch: fetchFn as any,
  83. })
  84. }
  85. describe("doStream", () => {
  86. test("should stream text deltas", async () => {
  87. const mockFetch = createMockFetch(FIXTURES.basicText)
  88. const model = createModel(mockFetch)
  89. const { stream } = await model.doStream({
  90. prompt: TEST_PROMPT,
  91. includeRawChunks: false,
  92. })
  93. const parts = await convertReadableStreamToArray(stream)
  94. // Filter to just the key events
  95. const textParts = parts.filter(
  96. (p) => p.type === "text-start" || p.type === "text-delta" || p.type === "text-end" || p.type === "finish",
  97. )
  98. expect(textParts).toMatchObject([
  99. { type: "text-start", id: "txt-0" },
  100. { type: "text-delta", id: "txt-0", delta: "Hello" },
  101. { type: "text-delta", id: "txt-0", delta: " world" },
  102. { type: "text-delta", id: "txt-0", delta: "!" },
  103. { type: "text-end", id: "txt-0" },
  104. { type: "finish", finishReason: "stop" },
  105. ])
  106. })
  107. test("should stream reasoning with tool calls and capture reasoning_opaque", async () => {
  108. const mockFetch = createMockFetch(FIXTURES.reasoningWithToolCalls)
  109. const model = createModel(mockFetch)
  110. const { stream } = await model.doStream({
  111. prompt: TEST_PROMPT,
  112. includeRawChunks: false,
  113. })
  114. const parts = await convertReadableStreamToArray(stream)
  115. // Check reasoning parts
  116. const reasoningParts = parts.filter(
  117. (p) => p.type === "reasoning-start" || p.type === "reasoning-delta" || p.type === "reasoning-end",
  118. )
  119. expect(reasoningParts[0]).toEqual({
  120. type: "reasoning-start",
  121. id: "reasoning-0",
  122. })
  123. expect(reasoningParts[1]).toMatchObject({
  124. type: "reasoning-delta",
  125. id: "reasoning-0",
  126. })
  127. expect((reasoningParts[1] as { delta: string }).delta).toContain("**Understanding Dayzee's Purpose**")
  128. expect(reasoningParts[2]).toMatchObject({
  129. type: "reasoning-delta",
  130. id: "reasoning-0",
  131. })
  132. expect((reasoningParts[2] as { delta: string }).delta).toContain("**Assessing Dayzee's Functionality**")
  133. // reasoning_opaque should be in reasoning-end providerMetadata
  134. const reasoningEnd = reasoningParts.find((p) => p.type === "reasoning-end")
  135. expect(reasoningEnd).toMatchObject({
  136. type: "reasoning-end",
  137. id: "reasoning-0",
  138. providerMetadata: {
  139. copilot: {
  140. reasoningOpaque: "4CUQ6696CwSXOdQ5rtvDimqA91tBzfmga4ieRbmZ5P67T2NLW3",
  141. },
  142. },
  143. })
  144. // Check tool calls
  145. const toolParts = parts.filter(
  146. (p) => p.type === "tool-input-start" || p.type === "tool-call" || p.type === "tool-input-end",
  147. )
  148. expect(toolParts).toContainEqual({
  149. type: "tool-input-start",
  150. id: "call_abc123",
  151. toolName: "read_file",
  152. })
  153. expect(toolParts).toContainEqual(
  154. expect.objectContaining({
  155. type: "tool-call",
  156. toolCallId: "call_abc123",
  157. toolName: "read_file",
  158. }),
  159. )
  160. expect(toolParts).toContainEqual({
  161. type: "tool-input-start",
  162. id: "call_def456",
  163. toolName: "read_file",
  164. })
  165. // Check finish
  166. const finish = parts.find((p) => p.type === "finish")
  167. expect(finish).toMatchObject({
  168. type: "finish",
  169. finishReason: "tool-calls",
  170. usage: {
  171. inputTokens: 19581,
  172. outputTokens: 53,
  173. },
  174. })
  175. })
  176. test("should handle reasoning_opaque that comes at end with text in between", async () => {
  177. const mockFetch = createMockFetch(FIXTURES.reasoningWithOpaqueAtEnd)
  178. const model = createModel(mockFetch)
  179. const { stream } = await model.doStream({
  180. prompt: TEST_PROMPT,
  181. includeRawChunks: false,
  182. })
  183. const parts = await convertReadableStreamToArray(stream)
  184. // Check that reasoning comes first
  185. const reasoningStart = parts.findIndex((p) => p.type === "reasoning-start")
  186. const textStart = parts.findIndex((p) => p.type === "text-start")
  187. expect(reasoningStart).toBeLessThan(textStart)
  188. // Check reasoning deltas
  189. const reasoningDeltas = parts.filter((p) => p.type === "reasoning-delta")
  190. expect(reasoningDeltas).toHaveLength(2)
  191. expect((reasoningDeltas[0] as { delta: string }).delta).toContain("**Analyzing the Inquiry's Nature**")
  192. expect((reasoningDeltas[1] as { delta: string }).delta).toContain("**Reconciling User's Input**")
  193. // Check text deltas
  194. const textDeltas = parts.filter((p) => p.type === "text-delta")
  195. expect(textDeltas).toHaveLength(2)
  196. expect((textDeltas[0] as { delta: string }).delta).toContain("I am Tidewave")
  197. expect((textDeltas[1] as { delta: string }).delta).toContain("How can I help you?")
  198. // reasoning-end should be emitted before text-start
  199. const reasoningEndIndex = parts.findIndex((p) => p.type === "reasoning-end")
  200. const textStartIndex = parts.findIndex((p) => p.type === "text-start")
  201. expect(reasoningEndIndex).toBeGreaterThan(-1)
  202. expect(reasoningEndIndex).toBeLessThan(textStartIndex)
  203. // In this fixture, reasoning_opaque comes AFTER content has started (in chunk 4)
  204. // So it arrives too late to be attached to reasoning-end. But it should still
  205. // be captured and included in the finish event's providerMetadata.
  206. const reasoningEnd = parts.find((p) => p.type === "reasoning-end")
  207. expect(reasoningEnd).toMatchObject({
  208. type: "reasoning-end",
  209. id: "reasoning-0",
  210. })
  211. // reasoning_opaque should be in the finish event's providerMetadata
  212. const finish = parts.find((p) => p.type === "finish")
  213. expect(finish).toMatchObject({
  214. type: "finish",
  215. finishReason: "stop",
  216. usage: {
  217. inputTokens: 5778,
  218. outputTokens: 59,
  219. },
  220. providerMetadata: {
  221. copilot: {
  222. reasoningOpaque: "/PMlTqxqSJZnUBDHgnnJKLVI4eZQ",
  223. },
  224. },
  225. })
  226. })
  227. test("should handle reasoning_opaque and content in the same chunk", async () => {
  228. const mockFetch = createMockFetch(FIXTURES.reasoningWithOpaqueAndContentSameChunk)
  229. const model = createModel(mockFetch)
  230. const { stream } = await model.doStream({
  231. prompt: TEST_PROMPT,
  232. includeRawChunks: false,
  233. })
  234. const parts = await convertReadableStreamToArray(stream)
  235. // The critical test: reasoning-end should come BEFORE text-start
  236. const reasoningEndIndex = parts.findIndex((p) => p.type === "reasoning-end")
  237. const textStartIndex = parts.findIndex((p) => p.type === "text-start")
  238. expect(reasoningEndIndex).toBeGreaterThan(-1)
  239. expect(textStartIndex).toBeGreaterThan(-1)
  240. expect(reasoningEndIndex).toBeLessThan(textStartIndex)
  241. // Check reasoning deltas
  242. const reasoningDeltas = parts.filter((p) => p.type === "reasoning-delta")
  243. expect(reasoningDeltas).toHaveLength(2)
  244. expect((reasoningDeltas[0] as { delta: string }).delta).toContain("**Understanding the Query's Nature**")
  245. expect((reasoningDeltas[1] as { delta: string }).delta).toContain("**Framing the Response's Core**")
  246. // reasoning_opaque should be in reasoning-end even though it came with content
  247. const reasoningEnd = parts.find((p) => p.type === "reasoning-end")
  248. expect(reasoningEnd).toMatchObject({
  249. type: "reasoning-end",
  250. id: "reasoning-0",
  251. providerMetadata: {
  252. copilot: {
  253. reasoningOpaque: "ExXaGwW7jBo39OXRe9EPoFGN1rOtLJBx",
  254. },
  255. },
  256. })
  257. // Check text deltas
  258. const textDeltas = parts.filter((p) => p.type === "text-delta")
  259. expect(textDeltas).toHaveLength(2)
  260. expect((textDeltas[0] as { delta: string }).delta).toContain("Of course. I'm thinking right now.")
  261. expect((textDeltas[1] as { delta: string }).delta).toContain("What's on your mind?")
  262. // Check finish
  263. const finish = parts.find((p) => p.type === "finish")
  264. expect(finish).toMatchObject({
  265. type: "finish",
  266. finishReason: "stop",
  267. })
  268. })
  269. test("should handle reasoning_opaque and content followed by tool calls", async () => {
  270. const mockFetch = createMockFetch(FIXTURES.reasoningWithOpaqueContentAndToolCalls)
  271. const model = createModel(mockFetch)
  272. const { stream } = await model.doStream({
  273. prompt: TEST_PROMPT,
  274. includeRawChunks: false,
  275. })
  276. const parts = await convertReadableStreamToArray(stream)
  277. // Check that reasoning comes first, then text, then tool calls
  278. const reasoningEndIndex = parts.findIndex((p) => p.type === "reasoning-end")
  279. const textStartIndex = parts.findIndex((p) => p.type === "text-start")
  280. const toolStartIndex = parts.findIndex((p) => p.type === "tool-input-start")
  281. expect(reasoningEndIndex).toBeGreaterThan(-1)
  282. expect(textStartIndex).toBeGreaterThan(-1)
  283. expect(toolStartIndex).toBeGreaterThan(-1)
  284. expect(reasoningEndIndex).toBeLessThan(textStartIndex)
  285. expect(textStartIndex).toBeLessThan(toolStartIndex)
  286. // Check reasoning content
  287. const reasoningDeltas = parts.filter((p) => p.type === "reasoning-delta")
  288. expect(reasoningDeltas).toHaveLength(1)
  289. expect((reasoningDeltas[0] as { delta: string }).delta).toContain("**Analyzing the Structure**")
  290. // reasoning_opaque should be in reasoning-end (comes with content in same chunk)
  291. const reasoningEnd = parts.find((p) => p.type === "reasoning-end")
  292. expect(reasoningEnd).toMatchObject({
  293. type: "reasoning-end",
  294. id: "reasoning-0",
  295. providerMetadata: {
  296. copilot: {
  297. reasoningOpaque: expect.stringContaining("WHOd3dYFnxEBOsKUXjbX6c2rJa0fS214"),
  298. },
  299. },
  300. })
  301. // Check text content
  302. const textDeltas = parts.filter((p) => p.type === "text-delta")
  303. expect(textDeltas).toHaveLength(1)
  304. expect((textDeltas[0] as { delta: string }).delta).toContain(
  305. "Okay, I need to check out the project's file structure.",
  306. )
  307. // Check tool call
  308. const toolParts = parts.filter(
  309. (p) => p.type === "tool-input-start" || p.type === "tool-call" || p.type === "tool-input-end",
  310. )
  311. expect(toolParts).toContainEqual({
  312. type: "tool-input-start",
  313. id: "call_MHxqRDd5WVo3NU8wUXRaMmc0MFE",
  314. toolName: "list_project_files",
  315. })
  316. expect(toolParts).toContainEqual(
  317. expect.objectContaining({
  318. type: "tool-call",
  319. toolCallId: "call_MHxqRDd5WVo3NU8wUXRaMmc0MFE",
  320. toolName: "list_project_files",
  321. }),
  322. )
  323. // Check finish
  324. const finish = parts.find((p) => p.type === "finish")
  325. expect(finish).toMatchObject({
  326. type: "finish",
  327. finishReason: "tool-calls",
  328. usage: {
  329. inputTokens: 3767,
  330. outputTokens: 19,
  331. },
  332. })
  333. })
  334. test("should emit reasoning-end before tool-input-start when reasoning goes directly to tool calls", async () => {
  335. const mockFetch = createMockFetch(FIXTURES.reasoningDirectlyToToolCalls)
  336. const model = createModel(mockFetch)
  337. const { stream } = await model.doStream({
  338. prompt: TEST_PROMPT,
  339. includeRawChunks: false,
  340. })
  341. const parts = await convertReadableStreamToArray(stream)
  342. // Critical check: reasoning-end MUST come before tool-input-start
  343. const reasoningEndIndex = parts.findIndex((p) => p.type === "reasoning-end")
  344. const toolStartIndex = parts.findIndex((p) => p.type === "tool-input-start")
  345. expect(reasoningEndIndex).toBeGreaterThan(-1)
  346. expect(toolStartIndex).toBeGreaterThan(-1)
  347. expect(reasoningEndIndex).toBeLessThan(toolStartIndex)
  348. // Check reasoning parts
  349. const reasoningDeltas = parts.filter((p) => p.type === "reasoning-delta")
  350. expect(reasoningDeltas).toHaveLength(2)
  351. expect((reasoningDeltas[0] as { delta: string }).delta).toContain("**Executing and Analyzing HTML**")
  352. expect((reasoningDeltas[1] as { delta: string }).delta).toContain("**Testing Project Contexts**")
  353. // reasoning_opaque should be in reasoning-end providerMetadata
  354. const reasoningEnd = parts.find((p) => p.type === "reasoning-end")
  355. expect(reasoningEnd).toMatchObject({
  356. type: "reasoning-end",
  357. id: "reasoning-0",
  358. providerMetadata: {
  359. copilot: {
  360. reasoningOpaque: "ytGNWFf2doK38peANDvm7whkLPKrd+Fv6/k34zEPBF6Qwitj4bTZT0FBXleydLb6",
  361. },
  362. },
  363. })
  364. // No text parts should exist
  365. const textParts = parts.filter((p) => p.type === "text-start" || p.type === "text-delta" || p.type === "text-end")
  366. expect(textParts).toHaveLength(0)
  367. // Check tool call
  368. const toolCall = parts.find((p) => p.type === "tool-call")
  369. expect(toolCall).toMatchObject({
  370. type: "tool-call",
  371. toolCallId: "call_MHw3RDhmT1J5Z3B6WlhpVjlveTc",
  372. toolName: "project_eval",
  373. })
  374. // Check finish
  375. const finish = parts.find((p) => p.type === "finish")
  376. expect(finish).toMatchObject({
  377. type: "finish",
  378. finishReason: "tool-calls",
  379. })
  380. })
  381. test("should include response metadata from first chunk", async () => {
  382. const mockFetch = createMockFetch(FIXTURES.basicText)
  383. const model = createModel(mockFetch)
  384. const { stream } = await model.doStream({
  385. prompt: TEST_PROMPT,
  386. includeRawChunks: false,
  387. })
  388. const parts = await convertReadableStreamToArray(stream)
  389. const metadata = parts.find((p) => p.type === "response-metadata")
  390. expect(metadata).toMatchObject({
  391. type: "response-metadata",
  392. id: "chatcmpl-123",
  393. modelId: "gemini-2.0-flash-001",
  394. })
  395. })
  396. test("should emit stream-start with warnings", async () => {
  397. const mockFetch = createMockFetch(FIXTURES.basicText)
  398. const model = createModel(mockFetch)
  399. const { stream } = await model.doStream({
  400. prompt: TEST_PROMPT,
  401. includeRawChunks: false,
  402. })
  403. const parts = await convertReadableStreamToArray(stream)
  404. const streamStart = parts.find((p) => p.type === "stream-start")
  405. expect(streamStart).toEqual({
  406. type: "stream-start",
  407. warnings: [],
  408. })
  409. })
  410. test("should include raw chunks when requested", async () => {
  411. const mockFetch = createMockFetch(FIXTURES.basicText)
  412. const model = createModel(mockFetch)
  413. const { stream } = await model.doStream({
  414. prompt: TEST_PROMPT,
  415. includeRawChunks: true,
  416. })
  417. const parts = await convertReadableStreamToArray(stream)
  418. const rawChunks = parts.filter((p) => p.type === "raw")
  419. expect(rawChunks.length).toBeGreaterThan(0)
  420. })
  421. })
  422. describe("request body", () => {
  423. test("should send tools in OpenAI format", async () => {
  424. let capturedBody: unknown
  425. const mockFetch = mock(async (_url: string, init?: RequestInit) => {
  426. capturedBody = JSON.parse(init?.body as string)
  427. return new Response(
  428. new ReadableStream({
  429. start(controller) {
  430. controller.enqueue(new TextEncoder().encode(`data: [DONE]\n\n`))
  431. controller.close()
  432. },
  433. }),
  434. { status: 200, headers: { "Content-Type": "text/event-stream" } },
  435. )
  436. })
  437. const model = createModel(mockFetch)
  438. await model.doStream({
  439. prompt: TEST_PROMPT,
  440. tools: [
  441. {
  442. type: "function",
  443. name: "get_weather",
  444. description: "Get the weather for a location",
  445. inputSchema: {
  446. type: "object",
  447. properties: {
  448. location: { type: "string" },
  449. },
  450. required: ["location"],
  451. },
  452. },
  453. ],
  454. includeRawChunks: false,
  455. })
  456. expect((capturedBody as { tools: unknown[] }).tools).toEqual([
  457. {
  458. type: "function",
  459. function: {
  460. name: "get_weather",
  461. description: "Get the weather for a location",
  462. parameters: {
  463. type: "object",
  464. properties: {
  465. location: { type: "string" },
  466. },
  467. required: ["location"],
  468. },
  469. },
  470. },
  471. ])
  472. })
  473. })