Generate grpc-js services and clients (#4199)

* Generate clientImpls and services for grpc-js.

Generate grpc-js services and clients (as opposed to the generic service definition)
The grpc-js clients are needed to connet to external gRPC services, ie the host bridge.
Switch the standalone gRPC service to use the grpc-js service defintions, these have the correct serialize/deserialize methods and fix the camel/snake case issue.

* Formatting

.gitignore

@@ -22,6 +22,7 @@ coverage
 *evals.env
 
 # Generated proto files
+src/generated/
 src/core/controller/*/methods.ts
 src/core/controller/*/index.ts
 src/core/controller/grpc-service-config.ts

proto/build-proto.js

@@ -10,25 +10,24 @@ import os from "os"
 
 import { createRequire } from "module"
 const require = createRequire(import.meta.url)
-const protoc = path.join(require.resolve("grpc-tools"), "../bin/protoc")
+const PROTOC = path.join(require.resolve("grpc-tools"), "../bin/protoc")
 
-const __filename = fileURLToPath(import.meta.url)
-const SCRIPT_DIR = path.dirname(__filename)
+const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url))
 const ROOT_DIR = path.resolve(SCRIPT_DIR, "..")
+
 const TS_OUT_DIR = path.join(ROOT_DIR, "src", "shared", "proto")
+const GRPC_JS_OUT_DIR = path.join(ROOT_DIR, "src", "generated", "grpc-js")
+const DESCRIPTOR_OUT_DIR = path.join(ROOT_DIR, "dist-standalone", "proto")
 
 const isWindows = process.platform === "win32"
-const tsProtoPlugin = isWindows
+const TS_PROTO_PLUGIN = isWindows
 	? path.join(ROOT_DIR, "node_modules", ".bin", "protoc-gen-ts_proto.cmd") // Use the .bin directory path for Windows
 	: require.resolve("ts-proto/protoc-gen-ts_proto")
 
 const TS_PROTO_OPTIONS = [
 	"env=node",
 	"esModuleInterop=true",
-
 	"outputIndex=true", // output an index file for each package which exports all protos in the package.
-	"outputServices=generic-definitions",
-
 	"useOptionals=messages", // Message fields are optional, scalars are not.
 	"useDate=false", // Timestamp fields will not be automatically converted to Date.
 ]
@@ -70,7 +69,9 @@ async function main() {
 	checkAppleSiliconCompatibility()
 
 	// Create output directories if they don't exist
-	await fs.mkdir(TS_OUT_DIR, { recursive: true })
+	for (const dir of [TS_OUT_DIR, GRPC_JS_OUT_DIR, DESCRIPTOR_OUT_DIR]) {
+		await fs.mkdir(dir, { recursive: true })
+	}
 
 	await cleanup()
 
@@ -81,28 +82,12 @@ async function main() {
 	const protoFiles = await globby("**/*.proto", { cwd: SCRIPT_DIR, realpath: true })
 	console.log(chalk.cyan(`Processing ${protoFiles.length} proto files from`), SCRIPT_DIR)
 
-	// Build the protoc command with proper path handling for cross-platform
-	const tsProtocCommand = [
-		protoc,
-		`--proto_path="${SCRIPT_DIR}"`,
-		`--plugin=protoc-gen-ts_proto="${tsProtoPlugin}"`,
-		`--ts_proto_out="${TS_OUT_DIR}"`,
-		`--ts_proto_opt=${TS_PROTO_OPTIONS.join(",")} `,
-		...protoFiles,
-	].join(" ")
-	try {
-		log_verbose(chalk.cyan(`Generating TypeScript code for:\n${protoFiles.join("\n")}...`))
-		execSync(tsProtocCommand, { stdio: "inherit" })
-	} catch (error) {
-		console.error(chalk.red("Error generating TypeScript for proto files:"), error)
-		process.exit(1)
-	}
+	tsProtoc(TS_OUT_DIR, protoFiles, ["outputServices=generic-definitions", ...TS_PROTO_OPTIONS])
+	tsProtoc(GRPC_JS_OUT_DIR, protoFiles, ["outputServices=grpc-js", ...TS_PROTO_OPTIONS])
 
-	const descriptorOutDir = path.join(ROOT_DIR, "dist-standalone", "proto")
-	await fs.mkdir(descriptorOutDir, { recursive: true })
-	const descriptorFile = path.join(descriptorOutDir, "descriptor_set.pb")
+	const descriptorFile = path.join(DESCRIPTOR_OUT_DIR, "descriptor_set.pb")
 	const descriptorProtocCommand = [
-		protoc,
+		PROTOC,
 		`--proto_path="${SCRIPT_DIR}"`,
 		`--descriptor_set_out="${descriptorFile}"`,
 		"--include_imports",
@@ -129,6 +114,25 @@ async function main() {
 	console.log(chalk.bold.blue("Finished Protocol Buffer code generation."))
 }
 
+async function tsProtoc(outDir, protoFiles, protoOptions) {
+	// Build the protoc command with proper path handling for cross-platform
+	const tsProtocCommand = [
+		PROTOC,
+		`--proto_path="${SCRIPT_DIR}"`,
+		`--plugin=protoc-gen-ts_proto="${TS_PROTO_PLUGIN}"`,
+		`--ts_proto_out="${outDir}"`,
+		`--ts_proto_opt=${protoOptions.join(",")} `,
+		...protoFiles,
+	].join(" ")
+	try {
+		log_verbose(chalk.cyan(`Generating TypeScript code in ${outDir} for:\n${protoFiles.join("\n")}...`))
+		execSync(tsProtocCommand, { stdio: "inherit" })
+	} catch (error) {
+		console.error(chalk.red("Error generating TypeScript for proto files:"), error)
+		process.exit(1)
+	}
+}
+
 /**
  * Generate a gRPC client configuration file for the webview
  * This eliminates the need for manual imports and client creation in grpc-client.ts
@@ -141,7 +145,7 @@ async function generateGrpcClientConfig() {
 	const serviceExports = []
 
 	// Process each service in the serviceNameMap
-	for (const [dirName, fullServiceName] of Object.entries(serviceNameMap)) {
+	for (const [dirName, _fullServiceName] of Object.entries(serviceNameMap)) {
 		const capitalizedName = dirName.charAt(0).toUpperCase() + dirName.slice(1)
 
 		// Add import statement

scripts/generate-server-setup.mjs

@@ -29,17 +29,17 @@ function generateHandlersAndExports() {
 		const dir = domain.charAt(0).toLowerCase() + domain.slice(1)
 		imports.push(`// ${domain} Service`)
 		handlerSetup.push(`    // ${domain} Service`)
-		handlerSetup.push(`    server.addService(proto.cline.${name}.service, {`)
+		handlerSetup.push(`    server.addService(cline.${name}Service, {`)
 		for (const [rpcName, rpc] of Object.entries(def.service)) {
 			imports.push(`import { ${rpcName} } from "../core/controller/${dir}/${rpcName}"`)
-			const requestType = "proto.cline." + rpc.requestType.type.name
+			const requestType = "cline." + rpc.requestType.type.name
 			if (rpc.requestStream) {
 				throw new Error("Request streaming is not supported")
 			}
 			if (rpc.responseStream) {
 				handlerSetup.push(`        ${rpcName}: wrapStreamingResponse<${requestType},void>(${rpcName}, controller),`)
 			} else {
-				const responseType = "proto.cline." + rpc.responseType.type.name
+				const responseType = "cline." + rpc.responseType.type.name
 				handlerSetup.push(`         ${rpcName}: wrapper<${requestType},${responseType}>(${rpcName}, controller),`)
 			}
 		}
@@ -60,14 +60,13 @@ const scriptName = path.basename(fileURLToPath(import.meta.url))
 let output = `// GENERATED CODE -- DO NOT EDIT!
 // Generated by ${scriptName}
 import * as grpc from "@grpc/grpc-js"
-import * as proto from "@/shared/proto"
+import { cline } from "../generated/grpc-js"
 import { Controller } from "../core/controller"
 import { GrpcHandlerWrapper, GrpcStreamingResponseHandlerWrapper } from "./grpc-types"
 
 ${imports}
 export function addServices(
 	server: grpc.Server,
-	proto: any,
 	controller: Controller,
 	wrapper: GrpcHandlerWrapper,
 	wrapStreamingResponse: GrpcStreamingResponseHandlerWrapper,

src/standalone/standalone.ts

@@ -5,7 +5,7 @@ import * as health from "grpc-health-check"
 import { activate } from "../extension"
 import { Controller } from "../core/controller"
 import { extensionContext, outputChannel, postMessage } from "./vscode-context"
-import { packageDefinition, proto, log, camelToSnakeCase, snakeToCamelCase } from "./utils"
+import { getPackageDefinition, log } from "./utils"
 import { GrpcHandler, GrpcStreamingResponseHandler } from "./grpc-types"
 import { addServices } from "./server-setup"
 import { StreamingResponseHandler } from "@/core/controller/grpc-handler"
@@ -22,10 +22,10 @@ function main() {
 	healthImpl.addToServer(server)
 
 	// Add all the handlers for the ProtoBus services to the server.
-	addServices(server, proto, controller, wrapHandler, wrapStreamingResponseHandler)
+	addServices(server, controller, wrapHandler, wrapStreamingResponseHandler)
 
 	// Set up reflection.
-	const reflection = new ReflectionService(packageDefinition)
+	const reflection = new ReflectionService(getPackageDefinition())
 	reflection.addToServer(server)
 
 	// Start the server.
@@ -58,10 +58,8 @@ function wrapHandler<TRequest, TResponse>(
 	return async (call: grpc.ServerUnaryCall<TRequest, TResponse>, callback: grpc.sendUnaryData<TResponse>) => {
 		try {
 			log(`gRPC request: ${call.getPath()}`)
-			const result = await handler(controller, snakeToCamelCase(call.request))
-			// The grpc-js serializer expects the proto message to be in the same
-			// case as the proto file. This is a work around until we find a solution.
-			callback(null, camelToSnakeCase(result))
+			const result = await handler(controller, call.request)
+			callback(null, result)
 		} catch (err: any) {
 			log(`gRPC handler error: ${call.getPath()}\n${err.stack}`)
 			callback({
@@ -83,9 +81,7 @@ function wrapStreamingResponseHandler<TRequest, TResponse>(
 
 			const responseHandler: StreamingResponseHandler = (response, isLast, sequenceNumber) => {
 				try {
-					// The grpc-js serializer expects the proto message to be in the same
-					// case as the proto file. This is a work around until we find a solution.
-					call.write(camelToSnakeCase(response)) // Use a bound version of call.write to maintain proper 'this' context
+					call.write(response) // Use a bound version of call.write to maintain proper 'this' context
 
 					if (isLast === true) {
 						log(`Closing stream for ${requestId}`)
@@ -96,7 +92,7 @@ function wrapStreamingResponseHandler<TRequest, TResponse>(
 					return Promise.reject(error)
 				}
 			}
-			await handler(controller, snakeToCamelCase(call.request), responseHandler, requestId)
+			await handler(controller, call.request, responseHandler, requestId)
 		} catch (err: any) {
 			log(`gRPC handler error: ${call.getPath()}\n${err.stack}`)
 			call.destroy({

src/standalone/utils.ts

@@ -8,58 +8,12 @@ const log = (...args: unknown[]) => {
 	console.log(`[${timestamp}]`, "#bot.cline.server.ts", ...args)
 }
 
-// Load service definitions.
-const descriptorSet = fs.readFileSync("proto/descriptor_set.pb")
-const clineDef = protoLoader.loadFileDescriptorSetFromBuffer(descriptorSet)
-const healthDef = protoLoader.loadSync(health.protoPath)
-const packageDefinition = { ...clineDef, ...healthDef }
-const proto = grpc.loadPackageDefinition(packageDefinition) as unknown
-
-// Helper function to convert camelCase to snake_case
-function camelToSnakeCase(obj: any): any {
-	if (obj === null || typeof obj !== "object") {
-		return obj
-	}
-
-	if (Array.isArray(obj)) {
-		return obj.map(camelToSnakeCase)
-	}
-
-	return Object.keys(obj).reduce((acc: any, key: string) => {
-		// Convert key from camelCase to snake_case
-		const snakeKey = key
-			.replace(/([A-Z])/g, "_$1")
-			.replace(/^_+/, "")
-			.toLowerCase()
-
-		// Convert value recursively if it's an object
-		const value = obj[key]
-		acc[snakeKey] = camelToSnakeCase(value)
-
-		return acc
-	}, {})
-}
-
-// Helper function to convert snake_case to camelCase
-function snakeToCamelCase(obj: any): any {
-	if (obj === null || typeof obj !== "object") {
-		return obj
-	}
-
-	if (Array.isArray(obj)) {
-		return obj.map(snakeToCamelCase)
-	}
-
-	return Object.keys(obj).reduce((acc: any, key: string) => {
-		// Convert key from snake_case to camelCase
-		const camelKey = key.replace(/_([a-z0-9])/g, (_, char) => char.toUpperCase())
-
-		// Convert value recursively if it's an object
-		const value = obj[key]
-		acc[camelKey] = snakeToCamelCase(value)
-
-		return acc
-	}, {})
+function getPackageDefinition() {
+	// Load service definitions.
+	const descriptorSet = fs.readFileSync("proto/descriptor_set.pb")
+	const clineDef = protoLoader.loadFileDescriptorSetFromBuffer(descriptorSet)
+	const healthDef = protoLoader.loadSync(health.protoPath)
+	const packageDefinition = { ...clineDef, ...healthDef }
+	return packageDefinition
 }
-
-export { packageDefinition, proto, log, camelToSnakeCase, snakeToCamelCase }
+export { getPackageDefinition, log }