Bare metal evals fixes (#8224)

Co-authored-by: Roo Code <[email protected]>

apps/web-evals/package.json

@@ -5,7 +5,7 @@
 	"scripts": {
 		"lint": "next lint --max-warnings 0",
 		"check-types": "tsc -b",
-		"dev": "scripts/check-services.sh && next dev",
+		"dev": "scripts/check-services.sh && next dev -p 3446",
 		"format": "prettier --write src",
 		"build": "next build",
 		"start": "next start",

packages/evals/README.md

@@ -95,7 +95,7 @@ By default, the evals system uses the following ports:
 
 - **PostgreSQL**: 5433 (external) → 5432 (internal)
 - **Redis**: 6380 (external) → 6379 (internal)
-- **Web Service**: 3446 (external) → 3000 (internal)
+- **Web Service**: 3446 (external) → 3446 (internal)
 
 These ports are configured to avoid conflicts with other services that might be running on the standard PostgreSQL (5432) and Redis (6379) ports.
 

packages/evals/docker-compose.yml

@@ -52,7 +52,7 @@ services:
             context: ../../
             dockerfile: packages/evals/Dockerfile.web
         ports:
-            - "${EVALS_WEB_PORT:-3446}:3000"
+            - "${EVALS_WEB_PORT:-3446}:3446"
         environment:
             - HOST_EXECUTION_METHOD=docker
         volumes:

packages/evals/scripts/setup.sh

@@ -12,7 +12,6 @@ build_extension() {
   echo "🔨 Building the Roo Code extension..."
   pnpm -w vsix -- --out ../bin/roo-code-$(git rev-parse --short HEAD).vsix || exit 1
   code --install-extension ../../bin/roo-code-$(git rev-parse --short HEAD).vsix || exit 1
-  cd evals
 }
 
 check_docker_services() {
@@ -377,7 +376,7 @@ fi
 
 echo -e "\n🚀 You're ready to rock and roll! \n"
 
-if ! nc -z localhost 3000; then
+if ! nc -z localhost 3446; then
   read -p "🌐 Would you like to start the evals web app? (Y/n): " start_evals
 
   if [[ "$start_evals" =~ ^[Yy]|^$ ]]; then
@@ -386,5 +385,5 @@ if ! nc -z localhost 3000; then
     echo "💡 You can start it anytime with 'pnpm --filter @roo-code/web-evals dev'."
   fi
 else
-  echo "👟 The evals web app is running at http://localhost:3000 (or http://localhost:3446 if using Docker)"
+  echo "👟 The evals web app is running at http://localhost:3446"
 fi

src/api/providers/__tests__/roo.spec.ts

@@ -36,26 +36,12 @@ vitest.mock("openai", () => {
 						return {
 							[Symbol.asyncIterator]: async function* () {
 								yield {
-									choices: [
-										{
-											delta: { content: "Test response" },
-											index: 0,
-										},
-									],
+									choices: [{ delta: { content: "Test response" }, index: 0 }],
 									usage: null,
 								}
 								yield {
-									choices: [
-										{
-											delta: {},
-											index: 0,
-										},
-									],
-									usage: {
-										prompt_tokens: 10,
-										completion_tokens: 5,
-										total_tokens: 15,
-									},
+									choices: [{ delta: {}, index: 0 }],
+									usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
 								}
 							},
 						}
@@ -73,6 +59,7 @@ const mockHasInstance = vitest.fn()
 // Create mock functions that we can control
 const mockGetSessionTokenFn = vitest.fn()
 const mockHasInstanceFn = vitest.fn()
+const mockOnFn = vitest.fn()
 
 vitest.mock("@roo-code/cloud", () => ({
 	CloudService: {
@@ -82,6 +69,8 @@ vitest.mock("@roo-code/cloud", () => ({
 				authService: {
 					getSessionToken: () => mockGetSessionTokenFn(),
 				},
+				on: vitest.fn(),
+				off: vitest.fn(),
 			}
 		},
 	},
@@ -409,11 +398,18 @@ describe("RooHandler", () => {
 		it("should handle undefined auth service gracefully", () => {
 			mockHasInstanceFn.mockReturnValue(true)
 			// Mock CloudService with undefined authService
-			const originalGetter = Object.getOwnPropertyDescriptor(CloudService, "instance")?.get
+			const originalGetSessionToken = mockGetSessionTokenFn.getMockImplementation()
+
+			// Temporarily make authService return undefined
+			mockGetSessionTokenFn.mockImplementation(() => undefined)
 
 			try {
 				Object.defineProperty(CloudService, "instance", {
-					get: () => ({ authService: undefined }),
+					get: () => ({
+						authService: undefined,
+						on: vitest.fn(),
+						off: vitest.fn(),
+					}),
 					configurable: true,
 				})
 
@@ -424,12 +420,11 @@ describe("RooHandler", () => {
 				const handler = new RooHandler(mockOptions)
 				expect(handler).toBeInstanceOf(RooHandler)
 			} finally {
-				// Always restore original getter, even if test fails
-				if (originalGetter) {
-					Object.defineProperty(CloudService, "instance", {
-						get: originalGetter,
-						configurable: true,
-					})
+				// Restore original mock implementation
+				if (originalGetSessionToken) {
+					mockGetSessionTokenFn.mockImplementation(originalGetSessionToken)
+				} else {
+					mockGetSessionTokenFn.mockReturnValue("test-session-token")
 				}
 			}
 		})

src/api/providers/roo.ts

@@ -1,22 +1,24 @@
 import { Anthropic } from "@anthropic-ai/sdk"
+import OpenAI from "openai"
 
-import { rooDefaultModelId, rooModels, type RooModelId } from "@roo-code/types"
+import { AuthState, rooDefaultModelId, rooModels, type RooModelId } from "@roo-code/types"
 import { CloudService } from "@roo-code/cloud"
 
 import type { ApiHandlerOptions } from "../../shared/api"
 import { ApiStream } from "../transform/stream"
 
 import type { ApiHandlerCreateMessageMetadata } from "../index"
+import { DEFAULT_HEADERS } from "./constants"
 import { BaseOpenAiCompatibleProvider } from "./base-openai-compatible-provider"
 
 export class RooHandler extends BaseOpenAiCompatibleProvider<RooModelId> {
+	private authStateListener?: (state: { state: AuthState }) => void
+
 	constructor(options: ApiHandlerOptions) {
-		// Get the session token if available, but don't throw if not.
-		// The server will handle authentication errors and return appropriate status codes.
-		let sessionToken = ""
+		let sessionToken: string | undefined = undefined
 
 		if (CloudService.hasInstance()) {
-			sessionToken = CloudService.instance.authService?.getSessionToken() || ""
+			sessionToken = CloudService.instance.authService?.getSessionToken()
 		}
 
 		// Always construct the handler, even without a valid token.
@@ -25,11 +27,39 @@ export class RooHandler extends BaseOpenAiCompatibleProvider<RooModelId> {
 			...options,
 			providerName: "Roo Code Cloud",
 			baseURL: process.env.ROO_CODE_PROVIDER_URL ?? "https://api.roocode.com/proxy/v1",
-			apiKey: sessionToken || "unauthenticated", // Use a placeholder if no token
+			apiKey: sessionToken || "unauthenticated", // Use a placeholder if no token.
 			defaultProviderModelId: rooDefaultModelId,
 			providerModels: rooModels,
 			defaultTemperature: 0.7,
 		})
+
+		if (CloudService.hasInstance()) {
+			const cloudService = CloudService.instance
+
+			this.authStateListener = (state: { state: AuthState }) => {
+				if (state.state === "active-session") {
+					this.client = new OpenAI({
+						baseURL: this.baseURL,
+						apiKey: cloudService.authService?.getSessionToken() ?? "unauthenticated",
+						defaultHeaders: DEFAULT_HEADERS,
+					})
+				} else if (state.state === "logged-out") {
+					this.client = new OpenAI({
+						baseURL: this.baseURL,
+						apiKey: "unauthenticated",
+						defaultHeaders: DEFAULT_HEADERS,
+					})
+				}
+			}
+
+			cloudService.on("auth-state-changed", this.authStateListener)
+		}
+	}
+
+	dispose() {
+		if (this.authStateListener && CloudService.hasInstance()) {
+			CloudService.instance.off("auth-state-changed", this.authStateListener)
+		}
 	}
 
 	override async *createMessage(

src/extension.ts

@@ -194,7 +194,7 @@ export async function activate(context: vscode.ExtensionContext) {
 	// Add to subscriptions for proper cleanup on deactivate.
 	context.subscriptions.push(cloudService)
 
-	// Trigger initial cloud profile sync now that CloudService is ready
+	// Trigger initial cloud profile sync now that CloudService is ready.
 	try {
 		await provider.initializeCloudProfileSyncWhenReady()
 	} catch (error) {