Run Testing platform within Test workflow [shadow] (#6273)

Run Testing platform within Test workflow [shadow]

.changeset/heavy-llamas-smile.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Run Testing platform within Test workflow

.github/workflows/test.yml

@@ -149,6 +149,63 @@ jobs:
                       exit 1
                   fi
 
+    test-platform-integration:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout code
+              uses: actions/checkout@v4
+
+            - name: Setup Node.js environment
+              uses: actions/setup-node@v4
+              with:
+                  node-version: 22
+
+            - name: Cache root dependencies
+              uses: actions/cache@v4
+              id: root-cache
+              with:
+                  path: node_modules
+                  key: ${{ runner.os }}-npm-${{ hashFiles('package-lock.json') }}
+
+            - name: Cache webview-ui dependencies
+              uses: actions/cache@v4
+              id: webview-cache
+              with:
+                  path: webview-ui/node_modules
+                  key: ${{ runner.os }}-npm-webview-${{ hashFiles('webview-ui/package-lock.json') }}
+
+            # Cache testing-platform dependencies
+            - name: Cache testing-platform dependencies
+              uses: actions/cache@v4
+              id: testing-platform-cache
+              with:
+                path: testing-platform/node_modules
+                key: ${{ runner.os }}-npm-testing-platform-${{ hashFiles('testing-platform/package-lock.json') }}
+
+            - name: Install root dependencies
+              if: steps.root-cache.outputs.cache-hit != 'true'
+              run: npm ci
+
+            - name: Install webview-ui dependencies
+              if: steps.webview-cache.outputs.cache-hit != 'true'
+              run: cd webview-ui && npm ci
+
+            - name: Compile standalone
+              run: npm run compile-standalone
+
+            - name: Install testing platform dependencies
+              if: steps.testing-platform-cache.outputs.cache-hit != 'true'
+              run: cd testing-platform && npm ci
+
+            - name: Running testing platform integration spec tests
+              continue-on-error: true
+              timeout-minutes: 5
+              # Temporarily wrapping the test command to always return a neutral exit code.
+              # This prevents the job from showing as failed and avoids distracting developers
+              # until the integration tests are ready to be enforced.
+              run: |
+                npm run test:tp-orchestrator -- tests/specs/ --count=1 || true
+
     coverage:
         needs: test
         runs-on: ubuntu-latest

package-lock.json

@@ -117,6 +117,7 @@
 				"rimraf": "^6.0.1",
 				"should": "^13.2.3",
 				"sinon": "^19.0.2",
+				"tree-kill": "^1.2.2",
 				"ts-node": "^10.9.2",
 				"ts-proto": "^2.6.1",
 				"tsconfig-paths": "^4.2.0",
@@ -14493,6 +14494,16 @@
 			"version": "0.3.9",
 			"license": "MIT/X11"
 		},
+		"node_modules/tree-kill": {
+			"version": "1.2.2",
+			"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
+			"integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
+			"dev": true,
+			"license": "MIT",
+			"bin": {
+				"tree-kill": "cli.js"
+			}
+		},
 		"node_modules/tree-sitter-wasms": {
 			"version": "0.1.11",
 			"license": "Unlicense"

package.json

@@ -418,6 +418,7 @@
 		"rimraf": "^6.0.1",
 		"should": "^13.2.3",
 		"sinon": "^19.0.2",
+		"tree-kill": "^1.2.2",
 		"ts-node": "^10.9.2",
 		"ts-proto": "^2.6.1",
 		"tsconfig-paths": "^4.2.0",

scripts/test-standalone-core-api-server.ts

@@ -28,14 +28,13 @@
  * Ideal for local development, testing, or lightweight E2E scenarios.
  */
 
+import * as fs from "node:fs"
 import { mkdtempSync, rmSync } from "node:fs"
 import * as os from "node:os"
 import { ChildProcess, execSync, spawn } from "child_process"
-import * as fs from "fs"
 import * as path from "path"
 import { ClineApiServerMock } from "../src/test/e2e/fixtures/server/index"
 
-// Configuration
 const PROTOBUS_PORT = process.env.PROTOBUS_PORT || "26040"
 const HOSTBRIDGE_PORT = process.env.HOSTBRIDGE_PORT || "26041"
 const WORKSPACE_DIR = process.env.WORKSPACE_DIR || process.cwd()
@@ -48,6 +47,8 @@ const distDir = process.env.CLINE_DIST_DIR || path.join(projectRoot, "dist-stand
 const clineCoreFile = process.env.CLINE_CORE_FILE || "cline-core.js"
 const coreFile = path.join(distDir, clineCoreFile)
 
+const childProcesses: ChildProcess[] = []
+
 async function main(): Promise<void> {
 	console.log("Starting Simple Cline gRPC Server...")
 	console.log(`Workspace: ${WORKSPACE_DIR}`)
@@ -75,30 +76,24 @@ async function main(): Promise<void> {
 		process.exit(1)
 	}
 
-	// Fixed extension directory
 	const extensionsDir = path.join(distDir, "vsce-extension")
-
-	// Create temporary directories like e2e tests
 	const userDataDir = mkdtempSync(path.join(os.tmpdir(), "vsce"))
 	const clineTestWorkspace = mkdtempSync(path.join(os.tmpdir(), "cline-test-workspace-"))
 
-	// Start hostbridge test server in background.
-	// We run it as a child process to emulate how the extension currently operates
 	console.log("Starting HostBridge test server...")
 	const hostbridge: ChildProcess = spawn("npx", ["tsx", path.join(__dirname, "test-hostbridge-server.ts")], {
 		stdio: "pipe",
-		detached: false,
 		env: {
 			...process.env,
 			TEST_HOSTBRIDGE_WORKSPACE_DIR: clineTestWorkspace,
 			HOST_BRIDGE_ADDRESS: `127.0.0.1:${HOSTBRIDGE_PORT}`,
 		},
 	})
+	childProcesses.push(hostbridge)
 
 	console.log(`Temp user data dir: ${userDataDir}`)
 	console.log(`Temp extensions dir: ${extensionsDir}`)
-
-	// Extract standalone.zip to the extensions directory
+	// Extract standalone.zip if needed
 	const standaloneZipPath = path.join(distDir, "standalone.zip")
 	if (!fs.existsSync(standaloneZipPath)) {
 		console.error(`standalone.zip not found at: ${standaloneZipPath}`)
@@ -116,8 +111,6 @@ async function main(): Promise<void> {
 		process.exit(1)
 	}
 
-	// Start the core service
-	// We run it as a child process to emulate how the extension currently operates
 	console.log("Starting Cline Core Service...")
 	const coreService: ChildProcess = spawn("node", [clineCoreFile], {
 		cwd: distDir,
@@ -127,28 +120,31 @@ async function main(): Promise<void> {
 			DEV_WORKSPACE_FOLDER: WORKSPACE_DIR,
 			PROTOBUS_ADDRESS: `127.0.0.1:${PROTOBUS_PORT}`,
 			HOST_BRIDGE_ADDRESS: `localhost:${HOSTBRIDGE_PORT}`,
-			E2E_TEST: E2E_TEST,
-			CLINE_ENVIRONMENT: CLINE_ENVIRONMENT,
+			E2E_TEST,
+			CLINE_ENVIRONMENT,
 			CLINE_DIR: userDataDir,
 			INSTALL_DIR: extensionsDir,
 		},
 		stdio: "inherit",
 	})
+	childProcesses.push(coreService)
+
+	const shutdown = async () => {
+		console.log("\nShutting down services...")
+
+		while (childProcesses.length > 0) {
+			const child = childProcesses.pop()
+			if (child && !child.killed) child.kill("SIGINT")
+		}
 
-	// Handle graceful shutdown
-	const shutdown = async (): Promise<void> => {
-		console.log(`\n Shutting down services...\n${userDataDir}\n${extensionsDir}\n${clineTestWorkspace}\n`)
-		hostbridge.kill()
-		coreService.kill()
 		await ClineApiServerMock.stopGlobalServer()
 
-		// Cleanup temp directories
 		try {
 			rmSync(userDataDir, { recursive: true, force: true })
 			rmSync(clineTestWorkspace, { recursive: true, force: true })
 			console.log("Cleaned up temporary directories")
-		} catch (error) {
-			console.warn("Failed to cleanup temp directories:", error)
+		} catch (err) {
+			console.warn("Failed to cleanup temp directories:", err)
 		}
 
 		process.exit(0)
@@ -159,24 +155,20 @@ async function main(): Promise<void> {
 
 	coreService.on("exit", (code) => {
 		console.log(`Core service exited with code ${code}`)
-		hostbridge.kill()
-		process.exit(code || 0)
+		shutdown()
 	})
-
 	hostbridge.on("exit", (code) => {
 		console.log(`HostBridge exited with code ${code}`)
-		coreService.kill()
-		process.exit(code || 0)
+		shutdown()
 	})
 
-	console.log("Cline gRPC Server is running!")
-	console.log(`Connect to: 127.0.0.1:${PROTOBUS_PORT}`)
+	console.log(`Cline gRPC Server is running on 127.0.0.1:${PROTOBUS_PORT}`)
 	console.log("Press Ctrl+C to stop")
 }
 
 if (require.main === module) {
-	main().catch((error) => {
-		console.error("Failed to start simple Cline server:", error)
+	main().catch((err) => {
+		console.error("Failed to start simple Cline server:", err)
 		process.exit(1)
 	})
 }

scripts/testing-platform-orchestrator.ts

@@ -24,53 +24,74 @@
 import { ChildProcess, spawn } from "child_process"
 import fs from "fs"
 import minimist from "minimist"
+import net from "net"
 import path from "path"
-
-const STANDALONE_GRPC_SERVER_PORT = process.env.STANDALONE_GRPC_SERVER_PORT || "26040"
-const SERVER_BOOT_DELAY = Number(process.env.SERVER_BOOT_DELAY) || 1300
+import kill from "tree-kill"
 
 let showServerLogs = false
 let fix = false
 
-function startServer(): Promise<ChildProcess> {
-	return new Promise((resolve, reject) => {
-		const server = spawn("npx", ["tsx", "scripts/test-standalone-core-api-server.ts"], {
-			stdio: showServerLogs ? "inherit" : "ignore",
-		})
+const STANDALONE_GRPC_SERVER_PORT = process.env.STANDALONE_GRPC_SERVER_PORT || "26040"
+const WAIT_SERVER_DEFAULT_TIMEOUT = 15000
+
+// Poll until port is accepting connections
+async function waitForPort(port: number, host = "127.0.0.1", timeout = 10000): Promise<void> {
+	const start = Date.now()
+	const waitForPortSleepMs = 100
+	while (Date.now() - start < timeout) {
+		await new Promise((res) => setTimeout(res, waitForPortSleepMs))
+		try {
+			await new Promise<void>((resolve, reject) => {
+				const socket = net.connect(port, host, () => {
+					socket.destroy()
+					resolve()
+				})
+				socket.on("error", reject)
+			})
+			return
+		} catch {
+			// try again
+		}
+	}
+	throw new Error(`Timeout waiting for ${host}:${port}`)
+}
 
-		server.once("error", reject)
+async function startServer(): Promise<{ server: ChildProcess; grpcPort: string }> {
+	const grpcPort = STANDALONE_GRPC_SERVER_PORT
 
-		setTimeout(() => {
-			if (server.killed) {
-				reject(new Error("Server died during startup"))
-			} else {
-				resolve(server)
-			}
-		}, SERVER_BOOT_DELAY)
+	const server = spawn("npx", ["tsx", "scripts/test-standalone-core-api-server.ts"], {
+		stdio: showServerLogs ? "inherit" : "pipe",
+		env: { ...process.env, STANDALONE_GRPC_SERVER_PORT: grpcPort },
 	})
+
+	// Wait for either the server to become ready or fail on spawn error
+	await Promise.race([
+		waitForPort(Number(grpcPort), "127.0.0.1", WAIT_SERVER_DEFAULT_TIMEOUT),
+		new Promise((_, reject) => server.once("error", reject)),
+	])
+
+	return { server, grpcPort }
 }
 
 function stopServer(server: ChildProcess): Promise<void> {
 	return new Promise((resolve) => {
-		server.once("exit", () => resolve())
-		server.kill("SIGINT")
-		setTimeout(() => {
-			if (!server.killed) {
-				server.kill("SIGKILL")
-				resolve()
-			}
-		}, 5000)
+		if (!server.pid) return resolve()
+
+		kill(server.pid, "SIGKILL", (err) => {
+			if (err) console.warn("Failed to kill server process:", err)
+			server.once("exit", () => resolve())
+		})
 	})
 }
 
-function runTestingPlatform(specFile: string): Promise<void> {
+function runTestingPlatform(specFile: string, grpcPort: string): Promise<void> {
 	return new Promise((resolve, reject) => {
 		const testProcess = spawn("npx", ["ts-node", "index.ts", specFile, ...(fix ? ["--fix"] : [])], {
 			cwd: path.join(process.cwd(), "testing-platform"),
 			stdio: "inherit",
 			env: {
 				...process.env,
-				STANDALONE_GRPC_SERVER_PORT,
+				STANDALONE_GRPC_SERVER_PORT: grpcPort,
 			},
 		})
 
@@ -82,9 +103,9 @@ function runTestingPlatform(specFile: string): Promise<void> {
 }
 
 async function runSpec(specFile: string): Promise<void> {
-	const server = await startServer()
+	const { server, grpcPort } = await startServer()
 	try {
-		await runTestingPlatform(specFile)
+		await runTestingPlatform(specFile, grpcPort)
 		console.log(`✅ ${path.basename(specFile)} passed`)
 	} finally {
 		await stopServer(server)
@@ -135,8 +156,7 @@ async function runAll(inputPath: string, count: number) {
 	console.log(`✅ Passed: ${success}`)
 	if (failure > 0) console.log(`❌ Failed: ${failure}`)
 	console.log(`📋 Total specs: ${specFiles.length} Total runs: ${specFiles.length * count}`)
-	const totalElapsed = ((Date.now() - totalStart) / 1000).toFixed(2)
-	console.log(`\n🏁 All runs completed in ${totalElapsed}s`)
+	console.log(`🏁 All runs completed in ${((Date.now() - totalStart) / 1000).toFixed(2)}s`)
 }
 
 async function main() {

src/test/e2e/fixtures/server/index.ts

@@ -512,8 +512,15 @@ export class ClineApiServerMock {
 		ClineApiServerMock.globalSockets.forEach((socket) => socket.destroy())
 		ClineApiServerMock.globalSockets.clear()
 
-		await new Promise<void>((resolve) => {
-			server.close(() => resolve())
+		await new Promise<void>((resolve, reject) => {
+			server.close((err) => {
+				if (err) {
+					console.error("Error closing server:", err)
+					reject(err)
+				}
+				log("Server closed successfully")
+				resolve()
+			})
 		})
 
 		ClineApiServerMock.globalSharedServer = null