test(e2e): isolate prompt tests with per-worker backend (#20464)

packages/app/e2e/actions.ts

@@ -312,10 +312,11 @@ export async function openSettings(page: Page) {
   return dialog
 }
 
-export async function seedProjects(page: Page, input: { directory: string; extra?: string[] }) {
+export async function seedProjects(page: Page, input: { directory: string; extra?: string[]; serverUrl?: string }) {
   await page.addInitScript(
     (args: { directory: string; serverUrl: string; extra: string[] }) => {
       const key = "opencode.global.dat:server"
+      const defaultKey = "opencode.settings.dat:defaultServerUrl"
       const raw = localStorage.getItem(key)
       const parsed = (() => {
         if (!raw) return undefined
@@ -331,6 +332,7 @@ export async function seedProjects(page: Page, input: { directory: string; extra
       const lastProject = store.lastProject && typeof store.lastProject === "object" ? store.lastProject : {}
       const projects = store.projects && typeof store.projects === "object" ? store.projects : {}
       const nextProjects = { ...(projects as Record<string, unknown>) }
+      const nextList = list.includes(args.serverUrl) ? list : [args.serverUrl, ...list]
 
       const add = (origin: string, directory: string) => {
         const current = nextProjects[origin]
@@ -356,17 +358,18 @@ export async function seedProjects(page: Page, input: { directory: string; extra
       localStorage.setItem(
         key,
         JSON.stringify({
-          list,
+          list: nextList,
           projects: nextProjects,
           lastProject,
         }),
       )
+      localStorage.setItem(defaultKey, args.serverUrl)
     },
-    { directory: input.directory, serverUrl, extra: input.extra ?? [] },
+    { directory: input.directory, serverUrl: input.serverUrl ?? serverUrl, extra: input.extra ?? [] },
   )
 }
 
-export async function createTestProject() {
+export async function createTestProject(input?: { serverUrl?: string }) {
   const root = await fs.mkdtemp(path.join(os.tmpdir(), "opencode-e2e-project-"))
   const id = `e2e-${path.basename(root)}`
 
@@ -381,7 +384,7 @@ export async function createTestProject() {
     stdio: "ignore",
   })
 
-  return resolveDirectory(root)
+  return resolveDirectory(root, input?.serverUrl)
 }
 
 export async function cleanupTestProject(directory: string) {
@@ -430,22 +433,22 @@ export async function waitSlug(page: Page, skip: string[] = []) {
   return next
 }
 
-export async function resolveSlug(slug: string) {
+export async function resolveSlug(slug: string, input?: { serverUrl?: string }) {
   const directory = base64Decode(slug)
   if (!directory) throw new Error(`Failed to decode workspace slug: ${slug}`)
-  const resolved = await resolveDirectory(directory)
+  const resolved = await resolveDirectory(directory, input?.serverUrl)
   return { directory: resolved, slug: base64Encode(resolved), raw: slug }
 }
 
-export async function waitDir(page: Page, directory: string) {
-  const target = await resolveDirectory(directory)
+export async function waitDir(page: Page, directory: string, input?: { serverUrl?: string }) {
+  const target = await resolveDirectory(directory, input?.serverUrl)
   await expect
     .poll(
       async () => {
         await assertHealthy(page, "waitDir")
         const slug = slugFromUrl(page.url())
         if (!slug) return ""
-        return resolveSlug(slug)
+        return resolveSlug(slug, input)
           .then((item) => item.directory)
           .catch(() => "")
       },
@@ -455,15 +458,15 @@ export async function waitDir(page: Page, directory: string) {
   return { directory: target, slug: base64Encode(target) }
 }
 
-export async function waitSession(page: Page, input: { directory: string; sessionID?: string }) {
-  const target = await resolveDirectory(input.directory)
+export async function waitSession(page: Page, input: { directory: string; sessionID?: string; serverUrl?: string }) {
+  const target = await resolveDirectory(input.directory, input.serverUrl)
   await expect
     .poll(
       async () => {
         await assertHealthy(page, "waitSession")
         const slug = slugFromUrl(page.url())
         if (!slug) return false
-        const resolved = await resolveSlug(slug).catch(() => undefined)
+        const resolved = await resolveSlug(slug, { serverUrl: input.serverUrl }).catch(() => undefined)
         if (!resolved || resolved.directory !== target) return false
         const current = sessionIDFromUrl(page.url())
         if (input.sessionID && current !== input.sessionID) return false
@@ -473,7 +476,7 @@ export async function waitSession(page: Page, input: { directory: string; sessio
         if (input.sessionID && (!state || state.sessionID !== input.sessionID)) return false
         if (!input.sessionID && state?.sessionID) return false
         if (state?.dir) {
-          const dir = await resolveDirectory(state.dir).catch(() => state.dir ?? "")
+          const dir = await resolveDirectory(state.dir, input.serverUrl).catch(() => state.dir ?? "")
           if (dir !== target) return false
         }
 
@@ -489,9 +492,9 @@ export async function waitSession(page: Page, input: { directory: string; sessio
   return { directory: target, slug: base64Encode(target) }
 }
 
-export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000) {
-  const sdk = createSdk(directory)
-  const target = await resolveDirectory(directory)
+export async function waitSessionSaved(directory: string, sessionID: string, timeout = 30_000, serverUrl?: string) {
+  const sdk = createSdk(directory, serverUrl)
+  const target = await resolveDirectory(directory, serverUrl)
 
   await expect
     .poll(
@@ -501,7 +504,7 @@ export async function waitSessionSaved(directory: string, sessionID: string, tim
           .then((x) => x.data)
           .catch(() => undefined)
         if (!data?.directory) return ""
-        return resolveDirectory(data.directory).catch(() => data.directory)
+        return resolveDirectory(data.directory, serverUrl).catch(() => data.directory)
       },
       { timeout },
     )
@@ -666,8 +669,9 @@ export async function cleanupSession(input: {
   sessionID: string
   directory?: string
   sdk?: ReturnType<typeof createSdk>
+  serverUrl?: string
 }) {
-  const sdk = input.sdk ?? (input.directory ? createSdk(input.directory) : undefined)
+  const sdk = input.sdk ?? (input.directory ? createSdk(input.directory, input.serverUrl) : undefined)
   if (!sdk) throw new Error("cleanupSession requires sdk or directory")
   await waitSessionIdle(sdk, input.sessionID, 5_000).catch(() => undefined)
   const current = await status(sdk, input.sessionID).catch(() => undefined)
@@ -1019,3 +1023,13 @@ export async function openWorkspaceMenu(page: Page, workspaceSlug: string) {
   await expect(menu).toBeVisible()
   return menu
 }
+
+export async function assistantText(sdk: ReturnType<typeof createSdk>, sessionID: string) {
+  const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
+  return messages
+    .filter((m) => m.info.role === "assistant")
+    .flatMap((m) => m.parts)
+    .filter((p) => p.type === "text")
+    .map((p) => p.text)
+    .join("\n")
+}

packages/app/e2e/backend.ts

@@ -0,0 +1,119 @@
+import { spawn } from "node:child_process"
+import fs from "node:fs/promises"
+import net from "node:net"
+import os from "node:os"
+import path from "node:path"
+import { fileURLToPath } from "node:url"
+
+type Handle = {
+  url: string
+  stop: () => Promise<void>
+}
+
+function freePort() {
+  return new Promise<number>((resolve, reject) => {
+    const server = net.createServer()
+    server.once("error", reject)
+    server.listen(0, () => {
+      const address = server.address()
+      if (!address || typeof address === "string") {
+        server.close(() => reject(new Error("Failed to acquire a free port")))
+        return
+      }
+      server.close((err) => {
+        if (err) reject(err)
+        else resolve(address.port)
+      })
+    })
+  })
+}
+
+async function waitForHealth(url: string, probe = "/global/health") {
+  const end = Date.now() + 120_000
+  let last = ""
+  while (Date.now() < end) {
+    try {
+      const res = await fetch(`${url}${probe}`)
+      if (res.ok) return
+      last = `status ${res.status}`
+    } catch (err) {
+      last = err instanceof Error ? err.message : String(err)
+    }
+    await new Promise((resolve) => setTimeout(resolve, 250))
+  }
+  throw new Error(`Timed out waiting for backend health at ${url}${probe}${last ? ` (${last})` : ""}`)
+}
+
+const LOG_CAP = 100
+
+function cap(input: string[]) {
+  if (input.length > LOG_CAP) input.splice(0, input.length - LOG_CAP)
+}
+
+function tail(input: string[]) {
+  return input.slice(-40).join("")
+}
+
+export async function startBackend(label: string): Promise<Handle> {
+  const port = await freePort()
+  const sandbox = await fs.mkdtemp(path.join(os.tmpdir(), `opencode-e2e-${label}-`))
+  const appDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..")
+  const repoDir = path.resolve(appDir, "../..")
+  const opencodeDir = path.join(repoDir, "packages", "opencode")
+  const env = {
+    ...process.env,
+    OPENCODE_DISABLE_SHARE: process.env.OPENCODE_DISABLE_SHARE ?? "true",
+    OPENCODE_DISABLE_LSP_DOWNLOAD: "true",
+    OPENCODE_DISABLE_DEFAULT_PLUGINS: "true",
+    OPENCODE_EXPERIMENTAL_DISABLE_FILEWATCHER: "true",
+    OPENCODE_TEST_HOME: path.join(sandbox, "home"),
+    XDG_DATA_HOME: path.join(sandbox, "share"),
+    XDG_CACHE_HOME: path.join(sandbox, "cache"),
+    XDG_CONFIG_HOME: path.join(sandbox, "config"),
+    XDG_STATE_HOME: path.join(sandbox, "state"),
+    OPENCODE_CLIENT: "app",
+    OPENCODE_STRICT_CONFIG_DEPS: "true",
+  } satisfies Record<string, string | undefined>
+  const out: string[] = []
+  const err: string[] = []
+  const proc = spawn(
+    "bun",
+    ["run", "--conditions=browser", "./src/index.ts", "serve", "--port", String(port), "--hostname", "127.0.0.1"],
+    {
+      cwd: opencodeDir,
+      env,
+      stdio: ["ignore", "pipe", "pipe"],
+    },
+  )
+  proc.stdout?.on("data", (chunk) => { out.push(String(chunk)); cap(out) })
+  proc.stderr?.on("data", (chunk) => { err.push(String(chunk)); cap(err) })
+
+  const url = `http://127.0.0.1:${port}`
+  try {
+    await waitForHealth(url)
+  } catch (error) {
+    proc.kill("SIGTERM")
+    await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined)
+    throw new Error(
+      [
+        `Failed to start isolated e2e backend for ${label}`,
+        error instanceof Error ? error.message : String(error),
+        tail(out),
+        tail(err),
+      ]
+        .filter(Boolean)
+        .join("\n"),
+    )
+  }
+
+  return {
+    url,
+    async stop() {
+      if (proc.exitCode === null) {
+        proc.kill("SIGTERM")
+        await new Promise((resolve) => proc.once("exit", () => resolve(undefined))).catch(() => undefined)
+      }
+      await fs.rm(sandbox, { recursive: true, force: true }).catch(() => undefined)
+    },
+  }
+}

packages/app/e2e/fixtures.ts

@@ -3,6 +3,7 @@ import { ManagedRuntime } from "effect"
 import type { E2EWindow } from "../src/testing/terminal"
 import type { Item, Reply, Usage } from "../../opencode/test/lib/llm-server"
 import { TestLLMServer } from "../../opencode/test/lib/llm-server"
+import { startBackend } from "./backend"
 import {
   healthPhase,
   cleanupSession,
@@ -19,6 +20,20 @@ import { createSdk, dirSlug, getWorktree, sessionPath } from "./utils"
 type LLMFixture = {
   url: string
   push: (...input: (Item | Reply)[]) => Promise<void>
+  pushMatch: (
+    match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
+    ...input: (Item | Reply)[]
+  ) => Promise<void>
+  textMatch: (
+    match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
+    value: string,
+    opts?: { usage?: Usage },
+  ) => Promise<void>
+  toolMatch: (
+    match: (hit: { url: URL; body: Record<string, unknown> }) => boolean,
+    name: string,
+    input: unknown,
+  ) => Promise<void>
   text: (value: string, opts?: { usage?: Usage }) => Promise<void>
   tool: (name: string, input: unknown) => Promise<void>
   toolHang: (name: string, input: unknown) => Promise<void>
@@ -46,32 +61,54 @@ const seedModel = (() => {
   }
 })()
 
+type ProjectHandle = {
+  directory: string
+  slug: string
+  gotoSession: (sessionID?: string) => Promise<void>
+  trackSession: (sessionID: string, directory?: string) => void
+  trackDirectory: (directory: string) => void
+  sdk: ReturnType<typeof createSdk>
+}
+
+type ProjectOptions = {
+  extra?: string[]
+  model?: { providerID: string; modelID: string }
+  setup?: (directory: string) => Promise<void>
+  beforeGoto?: (project: { directory: string; sdk: ReturnType<typeof createSdk> }) => Promise<void>
+}
+
 type TestFixtures = {
   llm: LLMFixture
   sdk: ReturnType<typeof createSdk>
   gotoSession: (sessionID?: string) => Promise<void>
-  withProject: <T>(
-    callback: (project: {
-      directory: string
-      slug: string
-      gotoSession: (sessionID?: string) => Promise<void>
-      trackSession: (sessionID: string, directory?: string) => void
-      trackDirectory: (directory: string) => void
-    }) => Promise<T>,
-    options?: {
-      extra?: string[]
-      model?: { providerID: string; modelID: string }
-      setup?: (directory: string) => Promise<void>
-    },
-  ) => Promise<T>
+  withProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
+  withBackendProject: <T>(callback: (project: ProjectHandle) => Promise<T>, options?: ProjectOptions) => Promise<T>
 }
 
 type WorkerFixtures = {
+  backend: {
+    url: string
+    sdk: (directory?: string) => ReturnType<typeof createSdk>
+  }
   directory: string
   slug: string
 }
 
 export const test = base.extend<TestFixtures, WorkerFixtures>({
+  backend: [
+    async ({}, use, workerInfo) => {
+      const handle = await startBackend(`w${workerInfo.workerIndex}`)
+      try {
+        await use({
+          url: handle.url,
+          sdk: (directory?: string) => createSdk(directory, handle.url),
+        })
+      } finally {
+        await handle.stop()
+      }
+    },
+    { scope: "worker" },
+  ],
   llm: async ({}, use) => {
     const rt = ManagedRuntime.make(TestLLMServer.layer)
     try {
@@ -79,6 +116,9 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
       await use({
         url: svc.url,
         push: (...input) => rt.runPromise(svc.push(...input)),
+        pushMatch: (match, ...input) => rt.runPromise(svc.pushMatch(match, ...input)),
+        textMatch: (match, value, opts) => rt.runPromise(svc.textMatch(match, value, opts)),
+        toolMatch: (match, name, input) => rt.runPromise(svc.toolMatch(match, name, input)),
         text: (value, opts) => rt.runPromise(svc.text(value, opts)),
         tool: (name, input) => rt.runPromise(svc.tool(name, input)),
         toolHang: (name, input) => rt.runPromise(svc.toolHang(name, input)),
@@ -146,44 +186,70 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
     await use(gotoSession)
   },
   withProject: async ({ page }, use) => {
-    await use(async (callback, options) => {
-      const root = await createTestProject()
-      const sessions = new Map<string, string>()
-      const dirs = new Set<string>()
-      await options?.setup?.(root)
-      await seedStorage(page, { directory: root, extra: options?.extra, model: options?.model })
-
-      const gotoSession = async (sessionID?: string) => {
-        await page.goto(sessionPath(root, sessionID))
-        await waitSession(page, { directory: root, sessionID })
-        const current = sessionIDFromUrl(page.url())
-        if (current) trackSession(current)
-      }
+    await use((callback, options) =>
+      runProject(page, callback, options),
+    )
+  },
+  withBackendProject: async ({ page, backend }, use) => {
+    await use((callback, options) =>
+      runProject(page, callback, { ...options, serverUrl: backend.url, sdk: backend.sdk }),
+    )
+  },
+})
 
-      const trackSession = (sessionID: string, directory?: string) => {
-        sessions.set(sessionID, directory ?? root)
-      }
+async function runProject<T>(
+  page: Page,
+  callback: (project: ProjectHandle) => Promise<T>,
+  options?: ProjectOptions & {
+    serverUrl?: string
+    sdk?: (directory?: string) => ReturnType<typeof createSdk>
+  },
+) {
+  const url = options?.serverUrl
+  const root = await createTestProject(url ? { serverUrl: url } : undefined)
+  const sdk = options?.sdk?.(root) ?? createSdk(root, url)
+  const sessions = new Map<string, string>()
+  const dirs = new Set<string>()
+  await options?.setup?.(root)
+  await seedStorage(page, {
+    directory: root,
+    extra: options?.extra,
+    model: options?.model,
+    serverUrl: url,
+  })
 
-      const trackDirectory = (directory: string) => {
-        if (directory !== root) dirs.add(directory)
-      }
+  const gotoSession = async (sessionID?: string) => {
+    await page.goto(sessionPath(root, sessionID))
+    await waitSession(page, { directory: root, sessionID, serverUrl: url })
+    const current = sessionIDFromUrl(page.url())
+    if (current) trackSession(current)
+  }
 
-      try {
-        await gotoSession()
-        const slug = await waitSlug(page)
-        return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory })
-      } finally {
-        setHealthPhase(page, "cleanup")
-        await Promise.allSettled(
-          Array.from(sessions, ([sessionID, directory]) => cleanupSession({ sessionID, directory })),
-        )
-        await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))
-        await cleanupTestProject(root)
-        setHealthPhase(page, "test")
-      }
-    })
-  },
-})
+  const trackSession = (sessionID: string, directory?: string) => {
+    sessions.set(sessionID, directory ?? root)
+  }
+
+  const trackDirectory = (directory: string) => {
+    if (directory !== root) dirs.add(directory)
+  }
+
+  try {
+    await options?.beforeGoto?.({ directory: root, sdk })
+    await gotoSession()
+    const slug = await waitSlug(page)
+    return await callback({ directory: root, slug, gotoSession, trackSession, trackDirectory, sdk })
+  } finally {
+    setHealthPhase(page, "cleanup")
+    await Promise.allSettled(
+      Array.from(sessions, ([sessionID, directory]) =>
+        cleanupSession({ sessionID, directory, serverUrl: url }),
+      ),
+    )
+    await Promise.allSettled(Array.from(dirs, (directory) => cleanupTestProject(directory)))
+    await cleanupTestProject(root)
+    setHealthPhase(page, "test")
+  }
+}
 
 async function seedStorage(
   page: Page,
@@ -191,6 +257,7 @@ async function seedStorage(
     directory: string
     extra?: string[]
     model?: { providerID: string; modelID: string }
+    serverUrl?: string
   },
 ) {
   await seedProjects(page, input)

packages/app/e2e/prompt/mock.ts

@@ -0,0 +1,46 @@
+import { createSdk } from "../utils"
+
+export const openaiModel = { providerID: "openai", modelID: "gpt-5.3-chat-latest" }
+
+type Hit = { body: Record<string, unknown> }
+
+export function bodyText(hit: Hit) {
+  return JSON.stringify(hit.body)
+}
+
+export function titleMatch(hit: Hit) {
+  return bodyText(hit).includes("Generate a title for this conversation")
+}
+
+export function promptMatch(token: string) {
+  return (hit: Hit) => bodyText(hit).includes(token)
+}
+
+export async function withMockOpenAI<T>(input: { serverUrl: string; llmUrl: string; fn: () => Promise<T> }) {
+  const sdk = createSdk(undefined, input.serverUrl)
+  const prev = await sdk.global.config.get().then((res) => res.data ?? {})
+
+  try {
+    await sdk.global.config.update({
+      config: {
+        ...prev,
+        model: `${openaiModel.providerID}/${openaiModel.modelID}`,
+        enabled_providers: ["openai"],
+        provider: {
+          ...prev.provider,
+          openai: {
+            ...prev.provider?.openai,
+            options: {
+              ...prev.provider?.openai?.options,
+              apiKey: "test-key",
+              baseURL: input.llmUrl,
+            },
+          },
+        },
+      },
+    })
+    return await input.fn()
+  } finally {
+    await sdk.global.config.update({ config: prev })
+  }
+}

packages/app/e2e/prompt/prompt-async.spec.ts

@@ -1,47 +1,53 @@
 import { test, expect } from "../fixtures"
 import { promptSelector } from "../selectors"
-import { cleanupSession, sessionIDFromUrl, withSession } from "../actions"
+import { assistantText, sessionIDFromUrl, withSession } from "../actions"
+import { openaiModel, promptMatch, withMockOpenAI } from "./mock"
 
 const text = (value: string | null) => (value ?? "").replace(/\u200B/g, "").trim()
 
 // Regression test for Issue #12453: the synchronous POST /message endpoint holds
 // the connection open while the agent works, causing "Failed to fetch" over
 // VPN/Tailscale. The fix switches to POST /prompt_async which returns immediately.
-test("prompt succeeds when sync message endpoint is unreachable", async ({ page, sdk, gotoSession }) => {
+test("prompt succeeds when sync message endpoint is unreachable", async ({
+  page,
+  llm,
+  backend,
+  withBackendProject,
+}) => {
   test.setTimeout(120_000)
 
   // Simulate Tailscale/VPN killing the long-lived sync connection
   await page.route("**/session/*/message", (route) => route.abort("connectionfailed"))
 
-  await gotoSession()
+  await withMockOpenAI({
+    serverUrl: backend.url,
+    llmUrl: llm.url,
+    fn: async () => {
+      const token = `E2E_ASYNC_${Date.now()}`
+      await llm.textMatch(promptMatch(token), token)
 
-  const token = `E2E_ASYNC_${Date.now()}`
-  await page.locator(promptSelector).click()
-  await page.keyboard.type(`Reply with exactly: ${token}`)
-  await page.keyboard.press("Enter")
+      await withBackendProject(
+        async (project) => {
+          await page.locator(promptSelector).click()
+          await page.keyboard.type(`Reply with exactly: ${token}`)
+          await page.keyboard.press("Enter")
 
-  await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
-  const sessionID = sessionIDFromUrl(page.url())!
+          await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
+          const sessionID = sessionIDFromUrl(page.url())!
+          project.trackSession(sessionID)
 
-  try {
-    // Agent response arrives via SSE despite sync endpoint being dead
-    await expect
-      .poll(
-        async () => {
-          const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
-          return messages
-            .filter((m) => m.info.role === "assistant")
-            .flatMap((m) => m.parts)
-            .filter((p) => p.type === "text")
-            .map((p) => p.text)
-            .join("\n")
+          await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
+
+          await expect
+            .poll(() => assistantText(project.sdk, sessionID), { timeout: 90_000 })
+            .toContain(token)
+        },
+        {
+          model: openaiModel,
         },
-        { timeout: 90_000 },
       )
-      .toContain(token)
-  } finally {
-    await cleanupSession({ sdk, sessionID })
-  }
+    },
+  })
 })
 
 test("failed prompt send restores the composer input", async ({ page, sdk, gotoSession }) => {

packages/app/e2e/prompt/prompt.spec.ts

@@ -1,44 +1,9 @@
-import fs from "node:fs/promises"
-import path from "node:path"
 import { test, expect } from "../fixtures"
 import { promptSelector } from "../selectors"
-import { sessionIDFromUrl } from "../actions"
-import { createSdk } from "../utils"
+import { assistantText, sessionIDFromUrl } from "../actions"
+import { openaiModel, promptMatch, titleMatch, withMockOpenAI } from "./mock"
 
-async function config(dir: string, url: string) {
-  await fs.writeFile(
-    path.join(dir, "opencode.json"),
-    JSON.stringify({
-      $schema: "https://opencode.ai/config.json",
-      enabled_providers: ["e2e-llm"],
-      provider: {
-        "e2e-llm": {
-          name: "E2E LLM",
-          npm: "@ai-sdk/openai-compatible",
-          env: [],
-          models: {
-            "test-model": {
-              name: "Test Model",
-              tool_call: true,
-              limit: { context: 128000, output: 32000 },
-            },
-          },
-          options: {
-            apiKey: "test-key",
-            baseURL: url,
-          },
-        },
-      },
-      agent: {
-        build: {
-          model: "e2e-llm/test-model",
-        },
-      },
-    }),
-  )
-}
-
-test("can send a prompt and receive a reply", async ({ page, llm, withProject }) => {
+test("can send a prompt and receive a reply", async ({ page, llm, backend, withBackendProject }) => {
   test.setTimeout(120_000)
 
   const pageErrors: string[] = []
@@ -48,48 +13,43 @@ test("can send a prompt and receive a reply", async ({ page, llm, withProject })
   page.on("pageerror", onPageError)
 
   try {
-    await withProject(
-      async (project) => {
-        const sdk = createSdk(project.directory)
+    await withMockOpenAI({
+      serverUrl: backend.url,
+      llmUrl: llm.url,
+      fn: async () => {
         const token = `E2E_OK_${Date.now()}`
 
-        await llm.text(token)
-        await project.gotoSession()
+        await llm.textMatch(titleMatch, "E2E Title")
+        await llm.textMatch(promptMatch(token), token)
 
-        const prompt = page.locator(promptSelector)
-        await prompt.click()
-        await page.keyboard.type(`Reply with exactly: ${token}`)
-        await page.keyboard.press("Enter")
+        await withBackendProject(
+          async (project) => {
+            const prompt = page.locator(promptSelector)
+            await prompt.click()
+            await page.keyboard.type(`Reply with exactly: ${token}`)
+            await page.keyboard.press("Enter")
 
-        await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
+            await expect(page).toHaveURL(/\/session\/[^/?#]+/, { timeout: 30_000 })
 
-        const sessionID = (() => {
-          const id = sessionIDFromUrl(page.url())
-          if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
-          return id
-        })()
-        project.trackSession(sessionID)
+            const sessionID = (() => {
+              const id = sessionIDFromUrl(page.url())
+              if (!id) throw new Error(`Failed to parse session id from url: ${page.url()}`)
+              return id
+            })()
+            project.trackSession(sessionID)
 
-        await expect
-          .poll(
-            async () => {
-              const messages = await sdk.session.messages({ sessionID, limit: 50 }).then((r) => r.data ?? [])
-              return messages
-                .filter((m) => m.info.role === "assistant")
-                .flatMap((m) => m.parts)
-                .filter((p) => p.type === "text")
-                .map((p) => p.text)
-                .join("\n")
-            },
-            { timeout: 30_000 },
-          )
-          .toContain(token)
-      },
-      {
-        model: { providerID: "e2e-llm", modelID: "test-model" },
-        setup: (dir) => config(dir, llm.url),
+            await expect.poll(() => llm.calls()).toBeGreaterThanOrEqual(1)
+
+            await expect
+              .poll(() => assistantText(project.sdk, sessionID), { timeout: 30_000 })
+              .toContain(token)
+          },
+          {
+            model: openaiModel,
+          },
+        )
       },
-    )
+    })
   } finally {
     page.off("pageerror", onPageError)
   }

packages/app/e2e/utils.ts

@@ -26,21 +26,21 @@ export const serverNamePattern = new RegExp(`(?:${serverNames.map(escape).join("
 export const modKey = process.platform === "darwin" ? "Meta" : "Control"
 export const terminalToggleKey = "Control+Backquote"
 
-export function createSdk(directory?: string) {
-  return createOpencodeClient({ baseUrl: serverUrl, directory, throwOnError: true })
+export function createSdk(directory?: string, baseUrl = serverUrl) {
+  return createOpencodeClient({ baseUrl, directory, throwOnError: true })
 }
 
-export async function resolveDirectory(directory: string) {
-  return createSdk(directory)
+export async function resolveDirectory(directory: string, baseUrl = serverUrl) {
+  return createSdk(directory, baseUrl)
     .path.get()
     .then((x) => x.data?.directory ?? directory)
 }
 
-export async function getWorktree() {
-  const sdk = createSdk()
+export async function getWorktree(baseUrl = serverUrl) {
+  const sdk = createSdk(undefined, baseUrl)
   const result = await sdk.path.get()
   const data = result.data
-  if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${serverUrl}/path`)
+  if (!data?.worktree) throw new Error(`Failed to resolve a worktree from ${baseUrl}/path`)
   return data.worktree
 }
 

packages/opencode/test/lib/llm-server.ts

@@ -20,6 +20,13 @@ type Hit = {
   body: Record<string, unknown>
 }
 
+type Match = (hit: Hit) => boolean
+
+type Queue = {
+  item: Item
+  match?: Match
+}
+
 type Wait = {
   count: number
   ready: Deferred.Deferred<void>
@@ -420,7 +427,7 @@ const reset = Effect.fn("TestLLMServer.reset")(function* (item: Sse) {
     for (const part of item.tail) res.write(line(part))
     res.destroy(new Error("connection reset"))
   })
-  yield* Effect.never
+  return yield* Effect.never
 })
 
 function fail(item: HttpError) {
@@ -581,6 +588,9 @@ namespace TestLLMServer {
   export interface Service {
     readonly url: string
     readonly push: (...input: (Item | Reply)[]) => Effect.Effect<void>
+    readonly pushMatch: (match: Match, ...input: (Item | Reply)[]) => Effect.Effect<void>
+    readonly textMatch: (match: Match, value: string, opts?: { usage?: Usage }) => Effect.Effect<void>
+    readonly toolMatch: (match: Match, name: string, input: unknown) => Effect.Effect<void>
     readonly text: (value: string, opts?: { usage?: Usage }) => Effect.Effect<void>
     readonly tool: (name: string, input: unknown) => Effect.Effect<void>
     readonly toolHang: (name: string, input: unknown) => Effect.Effect<void>
@@ -605,11 +615,15 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
       const router = yield* HttpRouter.HttpRouter
 
       let hits: Hit[] = []
-      let list: Item[] = []
+      let list: Queue[] = []
       let waits: Wait[] = []
 
       const queue = (...input: (Item | Reply)[]) => {
-        list = [...list, ...input.map(item)]
+        list = [...list, ...input.map((value) => ({ item: item(value) }))]
+      }
+
+      const queueMatch = (match: Match, ...input: (Item | Reply)[]) => {
+        list = [...list, ...input.map((value) => ({ item: item(value), match }))]
       }
 
       const notify = Effect.fnUntraced(function* () {
@@ -619,19 +633,21 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
         yield* Effect.forEach(ready, (item) => Deferred.succeed(item.ready, void 0))
       })
 
-      const pull = () => {
-        const first = list[0]
-        if (!first) return
-        list = list.slice(1)
-        return first
+      const pull = (hit: Hit) => {
+        const index = list.findIndex((entry) => !entry.match || entry.match(hit))
+        if (index === -1) return
+        const first = list[index]
+        list = [...list.slice(0, index), ...list.slice(index + 1)]
+        return first.item
       }
 
       const handle = Effect.fn("TestLLMServer.handle")(function* (mode: "chat" | "responses") {
         const req = yield* HttpServerRequest.HttpServerRequest
-        const next = pull()
-        if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
         const body = yield* req.json.pipe(Effect.orElseSucceed(() => ({})))
-        hits = [...hits, hit(req.originalUrl, body)]
+        const current = hit(req.originalUrl, body)
+        const next = pull(current)
+        if (!next) return HttpServerResponse.text("unexpected request", { status: 500 })
+        hits = [...hits, current]
         yield* notify()
         if (next.type !== "sse") return fail(next)
         if (mode === "responses") return send(responses(next, modelFrom(body)))
@@ -655,6 +671,21 @@ export class TestLLMServer extends ServiceMap.Service<TestLLMServer, TestLLMServ
         push: Effect.fn("TestLLMServer.push")(function* (...input: (Item | Reply)[]) {
           queue(...input)
         }),
+        pushMatch: Effect.fn("TestLLMServer.pushMatch")(function* (match: Match, ...input: (Item | Reply)[]) {
+          queueMatch(match, ...input)
+        }),
+        textMatch: Effect.fn("TestLLMServer.textMatch")(function* (
+          match: Match,
+          value: string,
+          opts?: { usage?: Usage },
+        ) {
+          const out = reply().text(value)
+          if (opts?.usage) out.usage(opts.usage)
+          queueMatch(match, out.stop().item())
+        }),
+        toolMatch: Effect.fn("TestLLMServer.toolMatch")(function* (match: Match, name: string, input: unknown) {
+          queueMatch(match, reply().tool(name, input).item())
+        }),
         text: Effect.fn("TestLLMServer.text")(function* (value: string, opts?: { usage?: Usage }) {
           const out = reply().text(value)
           if (opts?.usage) out.usage(opts.usage)