Evals (#2238)

* Evals

* Remove redundant line

* Fix tsc error

* Disable debug mode

* Add option to kill run

.dockerignore

@@ -1,45 +0,0 @@
-# Version control
-# .git/
-# .gitignore
-# .gitattributes
-# .git-blame-ignore-revs
-# .gitconfig
-
-# Build artifacts
-bin/
-dist/
-**/dist/
-out/
-**/out/
-
-# Dependencies
-node_modules/
-**/node_modules/
-
-# Test and development files
-coverage/
-**/.vscode-test/
-
-# Configuration files
-# .env*
-knip.json
-.husky/
-
-# CI/CD
-# .changeset/
-# .github/
-# ellipsis.yaml
-
-# OS specific
-.DS_Store
-
-# Logs
-logs/
-*.log
-
-# Nix
-# flake.lock
-# flake.nix
-
-# Monorepo
-benchmark/exercises/

.vscodeignore

@@ -5,7 +5,7 @@
 .vscode-test/**
 out/**
 out-integration/**
-benchmark/**
+evals/**
 e2e/**
 node_modules/**
 src/**

benchmark/.env.local.sample

@@ -1,2 +0,0 @@
-OPENROUTER_API_KEY=sk-or-v1-...
-POSTHOG_API_KEY=phc_...

benchmark/Dockerfile

@@ -1,89 +0,0 @@
-# docker build -f Dockerfile.base -t roo-code-benchmark-base ..
-# docker build -f Dockerfile -t roo-code-benchmark ..
-# docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark
-# docker exec -it $(docker ps --filter "ancestor=roo-code-benchmark" -q) /bin/bash
-
-FROM ubuntu:latest
-
-# Install dependencies
-RUN apt update && apt install -y sudo curl git vim jq
-
-# Create a `vscode` user
-RUN useradd -m vscode -s /bin/bash && \
-  echo "vscode ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/vscode && \
-  chmod 0440 /etc/sudoers.d/vscode
-
-# Install VS Code
-# https://code.visualstudio.com/docs/setup/linux
-RUN apt install -y wget gpg apt-transport-https
-RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
-RUN install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
-RUN echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" | tee /etc/apt/sources.list.d/vscode.list > /dev/null
-RUN rm -f packages.microsoft.gpg
-RUN apt update && apt install -y code
-
-# Install Xvfb
-RUN apt install -y xvfb
-
-# [cpp] Install cmake 3.28.3
-RUN apt install -y cmake
-
-# [go] Install Go 1.22.2
-RUN apt install -y golang-go
-
-# [java] Install Java 21
-RUN apt install -y default-jre
-
-# [javascript] Install Node.js v18.20.6
-RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
-RUN apt update && apt install -y nodejs
-RUN npm install -g corepack@latest
-
-# [python] Install Python 3.12.3 and uv 0.6.6
-RUN apt install -y python3 python3-venv python3-dev python3-pip
-
-# [rust] Install Rust 1.85
-RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
-RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
-
-WORKDIR /home/vscode
-USER vscode
-
-# Enable corepack and install pnpm for the vscode user
-RUN corepack enable
-RUN yes y | pnpm --version
-
-COPY benchmark/entrypoint.sh /usr/local/bin/entrypoint.sh
-
-# Copy and build dependencies
-COPY --chown=vscode:vscode package*.json /home/vscode/repo/
-COPY --chown=vscode:vscode webview-ui/package*.json /home/vscode/repo/webview-ui/
-COPY --chown=vscode:vscode e2e/package*.json /home/vscode/repo/e2e/
-COPY --chown=vscode:vscode benchmark/package*.json /home/vscode/repo/benchmark/
-WORKDIR /home/vscode/repo
-RUN npm run install:all
-
-# Copy and build benchmark runner
-COPY --chown=vscode:vscode . /home/vscode/repo
-WORKDIR /home/vscode/repo/benchmark
-RUN npm run build
-
-# Copy exercises
-WORKDIR /home/vscode
-RUN git clone https://github.com/cte/Roo-Code-Benchmark.git exercises
-
-# Prepare exercises
-WORKDIR /home/vscode/exercises/python
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-RUN /home/vscode/.local/bin/uv sync
-
-# Build web-ui
-WORKDIR /home/vscode/exercises/web-ui
-RUN echo "DB_FILE_NAME=file:/tmp/benchmarks.db" > .env
-RUN pnpm install
-RUN npx drizzle-kit push
-
-# Run web-ui
-EXPOSE 3000
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
-CMD ["/usr/bin/pnpm", "dev"]

benchmark/README.md

@@ -1,51 +0,0 @@
-# Benchmark Harness
-
-Configure ENV vars (OpenRouter, PostHog, etc):
-
-```sh
-cp .env.local.sample .env.local
-# Update ENV vars as needed.
-```
-
-Build and run a Docker image with the development environment needed to run the
-benchmarks (C++, Go, Java, Node.js, Python & Rust):
-
-```sh
-npm run docker:start
-```
-
-Run an exercise:
-
-```sh
-npm run docker:benchmark -- -e exercises/javascript/binary
-```
-
-Select and run an exercise:
-
-```sh
-npm run cli
-```
-
-Select and run an exercise for a specific language:
-
-```sh
-npm run cli -- run rust
-```
-
-Run all exercises for a language:
-
-```sh
-npm run cli -- run rust all
-```
-
-Run all exercises:
-
-```sh
-npm run cli -- run all
-```
-
-Run all exercises using a specific runId (useful for re-trying when an unexpected error occurs):
-
-```sh
-npm run cli -- run all --runId 1
-```

benchmark/entrypoint.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-
-npx drizzle-kit push
-exec "$@"

benchmark/package-lock.json

@@ -1,2493 +0,0 @@
-{
-	"name": "benchmark",
-	"version": "0.1.0",
-	"lockfileVersion": 3,
-	"requires": true,
-	"packages": {
-		"": {
-			"name": "benchmark",
-			"version": "0.1.0",
-			"devDependencies": {
-				"@vscode/test-electron": "^2.4.0",
-				"gluegun": "^5.1.2",
-				"tsx": "^4.19.3",
-				"typescript": "^5.4.5",
-				"yargs": "^17.7.2"
-			}
-		},
-		"node_modules/@babel/code-frame": {
-			"version": "7.26.2",
-			"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz",
-			"integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"@babel/helper-validator-identifier": "^7.25.9",
-				"js-tokens": "^4.0.0",
-				"picocolors": "^1.0.0"
-			},
-			"engines": {
-				"node": ">=6.9.0"
-			}
-		},
-		"node_modules/@babel/helper-validator-identifier": {
-			"version": "7.25.9",
-			"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz",
-			"integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6.9.0"
-			}
-		},
-		"node_modules/@esbuild/aix-ppc64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.0.tgz",
-			"integrity": "sha512-O7vun9Sf8DFjH2UtqK8Ku3LkquL9SZL8OLY1T5NZkA34+wG3OQF7cl4Ql8vdNzM6fzBbYfLaiRLIOZ+2FOCgBQ==",
-			"cpu": [
-				"ppc64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"aix"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/android-arm": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.0.tgz",
-			"integrity": "sha512-PTyWCYYiU0+1eJKmw21lWtC+d08JDZPQ5g+kFyxP0V+es6VPPSUhM6zk8iImp2jbV6GwjX4pap0JFbUQN65X1g==",
-			"cpu": [
-				"arm"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"android"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/android-arm64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.0.tgz",
-			"integrity": "sha512-grvv8WncGjDSyUBjN9yHXNt+cq0snxXbDxy5pJtzMKGmmpPxeAmAhWxXI+01lU5rwZomDgD3kJwulEnhTRUd6g==",
-			"cpu": [
-				"arm64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"android"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/android-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.0.tgz",
-			"integrity": "sha512-m/ix7SfKG5buCnxasr52+LI78SQ+wgdENi9CqyCXwjVR2X4Jkz+BpC3le3AoBPYTC9NHklwngVXvbJ9/Akhrfg==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"android"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/darwin-arm64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.0.tgz",
-			"integrity": "sha512-mVwdUb5SRkPayVadIOI78K7aAnPamoeFR2bT5nszFUZ9P8UpK4ratOdYbZZXYSqPKMHfS1wdHCJk1P1EZpRdvw==",
-			"cpu": [
-				"arm64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"darwin"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/darwin-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.0.tgz",
-			"integrity": "sha512-DgDaYsPWFTS4S3nWpFcMn/33ZZwAAeAFKNHNa1QN0rI4pUjgqf0f7ONmXf6d22tqTY+H9FNdgeaAa+YIFUn2Rg==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"darwin"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/freebsd-arm64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.0.tgz",
-			"integrity": "sha512-VN4ocxy6dxefN1MepBx/iD1dH5K8qNtNe227I0mnTRjry8tj5MRk4zprLEdG8WPyAPb93/e4pSgi1SoHdgOa4w==",
-			"cpu": [
-				"arm64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"freebsd"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/freebsd-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.0.tgz",
-			"integrity": "sha512-mrSgt7lCh07FY+hDD1TxiTyIHyttn6vnjesnPoVDNmDfOmggTLXRv8Id5fNZey1gl/V2dyVK1VXXqVsQIiAk+A==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"freebsd"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-arm": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.0.tgz",
-			"integrity": "sha512-vkB3IYj2IDo3g9xX7HqhPYxVkNQe8qTK55fraQyTzTX/fxaDtXiEnavv9geOsonh2Fd2RMB+i5cbhu2zMNWJwg==",
-			"cpu": [
-				"arm"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-arm64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz",
-			"integrity": "sha512-9QAQjTWNDM/Vk2bgBl17yWuZxZNQIF0OUUuPZRKoDtqF2k4EtYbpyiG5/Dk7nqeK6kIJWPYldkOcBqjXjrUlmg==",
-			"cpu": [
-				"arm64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-ia32": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.0.tgz",
-			"integrity": "sha512-43ET5bHbphBegyeqLb7I1eYn2P/JYGNmzzdidq/w0T8E2SsYL1U6un2NFROFRg1JZLTzdCoRomg8Rvf9M6W6Gg==",
-			"cpu": [
-				"ia32"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-loong64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.0.tgz",
-			"integrity": "sha512-fC95c/xyNFueMhClxJmeRIj2yrSMdDfmqJnyOY4ZqsALkDrrKJfIg5NTMSzVBr5YW1jf+l7/cndBfP3MSDpoHw==",
-			"cpu": [
-				"loong64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-mips64el": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.0.tgz",
-			"integrity": "sha512-nkAMFju7KDW73T1DdH7glcyIptm95a7Le8irTQNO/qtkoyypZAnjchQgooFUDQhNAy4iu08N79W4T4pMBwhPwQ==",
-			"cpu": [
-				"mips64el"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-ppc64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.0.tgz",
-			"integrity": "sha512-NhyOejdhRGS8Iwv+KKR2zTq2PpysF9XqY+Zk77vQHqNbo/PwZCzB5/h7VGuREZm1fixhs4Q/qWRSi5zmAiO4Fw==",
-			"cpu": [
-				"ppc64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-riscv64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.0.tgz",
-			"integrity": "sha512-5S/rbP5OY+GHLC5qXp1y/Mx//e92L1YDqkiBbO9TQOvuFXM+iDqUNG5XopAnXoRH3FjIUDkeGcY1cgNvnXp/kA==",
-			"cpu": [
-				"riscv64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-s390x": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.0.tgz",
-			"integrity": "sha512-XM2BFsEBz0Fw37V0zU4CXfcfuACMrppsMFKdYY2WuTS3yi8O1nFOhil/xhKTmE1nPmVyvQJjJivgDT+xh8pXJA==",
-			"cpu": [
-				"s390x"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/linux-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.0.tgz",
-			"integrity": "sha512-9yl91rHw/cpwMCNytUDxwj2XjFpxML0y9HAOH9pNVQDpQrBxHy01Dx+vaMu0N1CKa/RzBD2hB4u//nfc+Sd3Cw==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"linux"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/netbsd-arm64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.0.tgz",
-			"integrity": "sha512-RuG4PSMPFfrkH6UwCAqBzauBWTygTvb1nxWasEJooGSJ/NwRw7b2HOwyRTQIU97Hq37l3npXoZGYMy3b3xYvPw==",
-			"cpu": [
-				"arm64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"netbsd"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/netbsd-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.0.tgz",
-			"integrity": "sha512-jl+qisSB5jk01N5f7sPCsBENCOlPiS/xptD5yxOx2oqQfyourJwIKLRA2yqWdifj3owQZCL2sn6o08dBzZGQzA==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"netbsd"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/openbsd-arm64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.0.tgz",
-			"integrity": "sha512-21sUNbq2r84YE+SJDfaQRvdgznTD8Xc0oc3p3iW/a1EVWeNj/SdUCbm5U0itZPQYRuRTW20fPMWMpcrciH2EJw==",
-			"cpu": [
-				"arm64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"openbsd"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/openbsd-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.0.tgz",
-			"integrity": "sha512-2gwwriSMPcCFRlPlKx3zLQhfN/2WjJ2NSlg5TKLQOJdV0mSxIcYNTMhk3H3ulL/cak+Xj0lY1Ym9ysDV1igceg==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"openbsd"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/sunos-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.0.tgz",
-			"integrity": "sha512-bxI7ThgLzPrPz484/S9jLlvUAHYMzy6I0XiU1ZMeAEOBcS0VePBFxh1JjTQt3Xiat5b6Oh4x7UC7IwKQKIJRIg==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"sunos"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/win32-arm64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.0.tgz",
-			"integrity": "sha512-ZUAc2YK6JW89xTbXvftxdnYy3m4iHIkDtK3CLce8wg8M2L+YZhIvO1DKpxrd0Yr59AeNNkTiic9YLf6FTtXWMw==",
-			"cpu": [
-				"arm64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"win32"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/win32-ia32": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.0.tgz",
-			"integrity": "sha512-eSNxISBu8XweVEWG31/JzjkIGbGIJN/TrRoiSVZwZ6pkC6VX4Im/WV2cz559/TXLcYbcrDN8JtKgd9DJVIo8GA==",
-			"cpu": [
-				"ia32"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"win32"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@esbuild/win32-x64": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.0.tgz",
-			"integrity": "sha512-ZENoHJBxA20C2zFzh6AI4fT6RraMzjYw4xKWemRTRmRVtN9c5DcH9r/f2ihEkMjOW5eGgrwCslG/+Y/3bL+DHQ==",
-			"cpu": [
-				"x64"
-			],
-			"dev": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"win32"
-			],
-			"engines": {
-				"node": ">=18"
-			}
-		},
-		"node_modules/@types/parse-json": {
-			"version": "4.0.2",
-			"resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
-			"integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/@vscode/test-electron": {
-			"version": "2.4.1",
-			"resolved": "https://registry.npmjs.org/@vscode/test-electron/-/test-electron-2.4.1.tgz",
-			"integrity": "sha512-Gc6EdaLANdktQ1t+zozoBVRynfIsMKMc94Svu1QreOBC8y76x4tvaK32TljrLi1LI2+PK58sDVbL7ALdqf3VRQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"http-proxy-agent": "^7.0.2",
-				"https-proxy-agent": "^7.0.5",
-				"jszip": "^3.10.1",
-				"ora": "^7.0.1",
-				"semver": "^7.6.2"
-			},
-			"engines": {
-				"node": ">=16"
-			}
-		},
-		"node_modules/agent-base": {
-			"version": "7.1.3",
-			"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz",
-			"integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">= 14"
-			}
-		},
-		"node_modules/ansi-colors": {
-			"version": "4.1.3",
-			"resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz",
-			"integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/ansi-regex": {
-			"version": "6.1.0",
-			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz",
-			"integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=12"
-			},
-			"funding": {
-				"url": "https://github.com/chalk/ansi-regex?sponsor=1"
-			}
-		},
-		"node_modules/ansi-styles": {
-			"version": "4.3.0",
-			"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-			"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"color-convert": "^2.0.1"
-			},
-			"engines": {
-				"node": ">=8"
-			},
-			"funding": {
-				"url": "https://github.com/chalk/ansi-styles?sponsor=1"
-			}
-		},
-		"node_modules/apisauce": {
-			"version": "2.1.6",
-			"resolved": "https://registry.npmjs.org/apisauce/-/apisauce-2.1.6.tgz",
-			"integrity": "sha512-MdxR391op/FucS2YQRfB/NMRyCnHEPDd4h17LRIuVYi0BpGmMhpxc0shbOpfs5ahABuBEffNCGal5EcsydbBWg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"axios": "^0.21.4"
-			}
-		},
-		"node_modules/app-module-path": {
-			"version": "2.2.0",
-			"resolved": "https://registry.npmjs.org/app-module-path/-/app-module-path-2.2.0.tgz",
-			"integrity": "sha512-gkco+qxENJV+8vFcDiiFhuoSvRXb2a/QPqpSoWhVz829VNJfOTnELbBmPmNKFxf3xdNnw4DWCkzkDaavcX/1YQ==",
-			"dev": true,
-			"license": "BSD-2-Clause"
-		},
-		"node_modules/async": {
-			"version": "3.2.6",
-			"resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz",
-			"integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/axios": {
-			"version": "0.21.4",
-			"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
-			"integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"follow-redirects": "^1.14.0"
-			}
-		},
-		"node_modules/balanced-match": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
-			"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/base64-js": {
-			"version": "1.5.1",
-			"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-			"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
-			"dev": true,
-			"funding": [
-				{
-					"type": "github",
-					"url": "https://github.com/sponsors/feross"
-				},
-				{
-					"type": "patreon",
-					"url": "https://www.patreon.com/feross"
-				},
-				{
-					"type": "consulting",
-					"url": "https://feross.org/support"
-				}
-			],
-			"license": "MIT"
-		},
-		"node_modules/bl": {
-			"version": "5.1.0",
-			"resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz",
-			"integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"buffer": "^6.0.3",
-				"inherits": "^2.0.4",
-				"readable-stream": "^3.4.0"
-			}
-		},
-		"node_modules/bl/node_modules/readable-stream": {
-			"version": "3.6.2",
-			"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-			"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"inherits": "^2.0.3",
-				"string_decoder": "^1.1.1",
-				"util-deprecate": "^1.0.1"
-			},
-			"engines": {
-				"node": ">= 6"
-			}
-		},
-		"node_modules/brace-expansion": {
-			"version": "1.1.11",
-			"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-			"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"balanced-match": "^1.0.0",
-				"concat-map": "0.0.1"
-			}
-		},
-		"node_modules/buffer": {
-			"version": "6.0.3",
-			"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
-			"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
-			"dev": true,
-			"funding": [
-				{
-					"type": "github",
-					"url": "https://github.com/sponsors/feross"
-				},
-				{
-					"type": "patreon",
-					"url": "https://www.patreon.com/feross"
-				},
-				{
-					"type": "consulting",
-					"url": "https://feross.org/support"
-				}
-			],
-			"license": "MIT",
-			"dependencies": {
-				"base64-js": "^1.3.1",
-				"ieee754": "^1.2.1"
-			}
-		},
-		"node_modules/callsites": {
-			"version": "3.1.0",
-			"resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz",
-			"integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/chalk": {
-			"version": "4.1.2",
-			"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-			"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-styles": "^4.1.0",
-				"supports-color": "^7.1.0"
-			},
-			"engines": {
-				"node": ">=10"
-			},
-			"funding": {
-				"url": "https://github.com/chalk/chalk?sponsor=1"
-			}
-		},
-		"node_modules/cli-cursor": {
-			"version": "4.0.0",
-			"resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz",
-			"integrity": "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"restore-cursor": "^4.0.0"
-			},
-			"engines": {
-				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/cli-spinners": {
-			"version": "2.9.2",
-			"resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz",
-			"integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/cli-table3": {
-			"version": "0.6.0",
-			"resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.0.tgz",
-			"integrity": "sha512-gnB85c3MGC7Nm9I/FkiasNBOKjOiO1RNuXXarQms37q4QMpWdlbBgD/VnOStA2faG1dpXMv31RFApjX1/QdgWQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"object-assign": "^4.1.0",
-				"string-width": "^4.2.0"
-			},
-			"engines": {
-				"node": "10.* || >= 12.*"
-			},
-			"optionalDependencies": {
-				"colors": "^1.1.2"
-			}
-		},
-		"node_modules/cliui": {
-			"version": "8.0.1",
-			"resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz",
-			"integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"string-width": "^4.2.0",
-				"strip-ansi": "^6.0.1",
-				"wrap-ansi": "^7.0.0"
-			},
-			"engines": {
-				"node": ">=12"
-			}
-		},
-		"node_modules/cliui/node_modules/ansi-regex": {
-			"version": "5.0.1",
-			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-			"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/cliui/node_modules/strip-ansi": {
-			"version": "6.0.1",
-			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-			"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-regex": "^5.0.1"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/clone": {
-			"version": "1.0.4",
-			"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
-			"integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=0.8"
-			}
-		},
-		"node_modules/color-convert": {
-			"version": "2.0.1",
-			"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-			"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"color-name": "~1.1.4"
-			},
-			"engines": {
-				"node": ">=7.0.0"
-			}
-		},
-		"node_modules/color-name": {
-			"version": "1.1.4",
-			"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-			"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/colors": {
-			"version": "1.4.0",
-			"resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz",
-			"integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=0.1.90"
-			}
-		},
-		"node_modules/concat-map": {
-			"version": "0.0.1",
-			"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-			"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/core-util-is": {
-			"version": "1.0.3",
-			"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
-			"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/cosmiconfig": {
-			"version": "7.0.1",
-			"resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.0.1.tgz",
-			"integrity": "sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"@types/parse-json": "^4.0.0",
-				"import-fresh": "^3.2.1",
-				"parse-json": "^5.0.0",
-				"path-type": "^4.0.0",
-				"yaml": "^1.10.0"
-			},
-			"engines": {
-				"node": ">=10"
-			}
-		},
-		"node_modules/cross-spawn": {
-			"version": "7.0.3",
-			"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
-			"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"path-key": "^3.1.0",
-				"shebang-command": "^2.0.0",
-				"which": "^2.0.1"
-			},
-			"engines": {
-				"node": ">= 8"
-			}
-		},
-		"node_modules/debug": {
-			"version": "4.4.0",
-			"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
-			"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ms": "^2.1.3"
-			},
-			"engines": {
-				"node": ">=6.0"
-			},
-			"peerDependenciesMeta": {
-				"supports-color": {
-					"optional": true
-				}
-			}
-		},
-		"node_modules/defaults": {
-			"version": "1.0.4",
-			"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
-			"integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"clone": "^1.0.2"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/eastasianwidth": {
-			"version": "0.2.0",
-			"resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
-			"integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/ejs": {
-			"version": "3.1.8",
-			"resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.8.tgz",
-			"integrity": "sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==",
-			"dev": true,
-			"license": "Apache-2.0",
-			"dependencies": {
-				"jake": "^10.8.5"
-			},
-			"bin": {
-				"ejs": "bin/cli.js"
-			},
-			"engines": {
-				"node": ">=0.10.0"
-			}
-		},
-		"node_modules/emoji-regex": {
-			"version": "8.0.0",
-			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
-			"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/enquirer": {
-			"version": "2.3.6",
-			"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
-			"integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-colors": "^4.1.1"
-			},
-			"engines": {
-				"node": ">=8.6"
-			}
-		},
-		"node_modules/error-ex": {
-			"version": "1.3.2",
-			"resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
-			"integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"is-arrayish": "^0.2.1"
-			}
-		},
-		"node_modules/esbuild": {
-			"version": "0.25.0",
-			"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.0.tgz",
-			"integrity": "sha512-BXq5mqc8ltbaN34cDqWuYKyNhX8D/Z0J1xdtdQ8UcIIIyJyz+ZMKUt58tF3SrZ85jcfN/PZYhjR5uDQAYNVbuw==",
-			"dev": true,
-			"hasInstallScript": true,
-			"license": "MIT",
-			"bin": {
-				"esbuild": "bin/esbuild"
-			},
-			"engines": {
-				"node": ">=18"
-			},
-			"optionalDependencies": {
-				"@esbuild/aix-ppc64": "0.25.0",
-				"@esbuild/android-arm": "0.25.0",
-				"@esbuild/android-arm64": "0.25.0",
-				"@esbuild/android-x64": "0.25.0",
-				"@esbuild/darwin-arm64": "0.25.0",
-				"@esbuild/darwin-x64": "0.25.0",
-				"@esbuild/freebsd-arm64": "0.25.0",
-				"@esbuild/freebsd-x64": "0.25.0",
-				"@esbuild/linux-arm": "0.25.0",
-				"@esbuild/linux-arm64": "0.25.0",
-				"@esbuild/linux-ia32": "0.25.0",
-				"@esbuild/linux-loong64": "0.25.0",
-				"@esbuild/linux-mips64el": "0.25.0",
-				"@esbuild/linux-ppc64": "0.25.0",
-				"@esbuild/linux-riscv64": "0.25.0",
-				"@esbuild/linux-s390x": "0.25.0",
-				"@esbuild/linux-x64": "0.25.0",
-				"@esbuild/netbsd-arm64": "0.25.0",
-				"@esbuild/netbsd-x64": "0.25.0",
-				"@esbuild/openbsd-arm64": "0.25.0",
-				"@esbuild/openbsd-x64": "0.25.0",
-				"@esbuild/sunos-x64": "0.25.0",
-				"@esbuild/win32-arm64": "0.25.0",
-				"@esbuild/win32-ia32": "0.25.0",
-				"@esbuild/win32-x64": "0.25.0"
-			}
-		},
-		"node_modules/escalade": {
-			"version": "3.2.0",
-			"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
-			"integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/escape-string-regexp": {
-			"version": "1.0.5",
-			"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-			"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=0.8.0"
-			}
-		},
-		"node_modules/execa": {
-			"version": "5.1.1",
-			"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
-			"integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"cross-spawn": "^7.0.3",
-				"get-stream": "^6.0.0",
-				"human-signals": "^2.1.0",
-				"is-stream": "^2.0.0",
-				"merge-stream": "^2.0.0",
-				"npm-run-path": "^4.0.1",
-				"onetime": "^5.1.2",
-				"signal-exit": "^3.0.3",
-				"strip-final-newline": "^2.0.0"
-			},
-			"engines": {
-				"node": ">=10"
-			},
-			"funding": {
-				"url": "https://github.com/sindresorhus/execa?sponsor=1"
-			}
-		},
-		"node_modules/filelist": {
-			"version": "1.0.4",
-			"resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",
-			"integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==",
-			"dev": true,
-			"license": "Apache-2.0",
-			"dependencies": {
-				"minimatch": "^5.0.1"
-			}
-		},
-		"node_modules/filelist/node_modules/brace-expansion": {
-			"version": "2.0.1",
-			"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-			"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"balanced-match": "^1.0.0"
-			}
-		},
-		"node_modules/filelist/node_modules/minimatch": {
-			"version": "5.1.6",
-			"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
-			"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"brace-expansion": "^2.0.1"
-			},
-			"engines": {
-				"node": ">=10"
-			}
-		},
-		"node_modules/follow-redirects": {
-			"version": "1.15.9",
-			"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz",
-			"integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==",
-			"dev": true,
-			"funding": [
-				{
-					"type": "individual",
-					"url": "https://github.com/sponsors/RubenVerborgh"
-				}
-			],
-			"license": "MIT",
-			"engines": {
-				"node": ">=4.0"
-			},
-			"peerDependenciesMeta": {
-				"debug": {
-					"optional": true
-				}
-			}
-		},
-		"node_modules/fs-jetpack": {
-			"version": "4.3.1",
-			"resolved": "https://registry.npmjs.org/fs-jetpack/-/fs-jetpack-4.3.1.tgz",
-			"integrity": "sha512-dbeOK84F6BiQzk2yqqCVwCPWTxAvVGJ3fMQc6E2wuEohS28mR6yHngbrKuVCK1KHRx/ccByDylqu4H5PCP2urQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"minimatch": "^3.0.2",
-				"rimraf": "^2.6.3"
-			}
-		},
-		"node_modules/fs.realpath": {
-			"version": "1.0.0",
-			"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-			"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
-			"dev": true,
-			"license": "ISC"
-		},
-		"node_modules/fsevents": {
-			"version": "2.3.3",
-			"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
-			"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
-			"dev": true,
-			"hasInstallScript": true,
-			"license": "MIT",
-			"optional": true,
-			"os": [
-				"darwin"
-			],
-			"engines": {
-				"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
-			}
-		},
-		"node_modules/get-caller-file": {
-			"version": "2.0.5",
-			"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
-			"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
-			"dev": true,
-			"license": "ISC",
-			"engines": {
-				"node": "6.* || 8.* || >= 10.*"
-			}
-		},
-		"node_modules/get-stream": {
-			"version": "6.0.1",
-			"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
-			"integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=10"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/get-tsconfig": {
-			"version": "4.10.0",
-			"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.10.0.tgz",
-			"integrity": "sha512-kGzZ3LWWQcGIAmg6iWvXn0ei6WDtV26wzHRMwDSzmAbcXrTEXxHy6IehI6/4eT6VRKyMP1eF1VqwrVUmE/LR7A==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"resolve-pkg-maps": "^1.0.0"
-			},
-			"funding": {
-				"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
-			}
-		},
-		"node_modules/glob": {
-			"version": "7.2.3",
-			"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
-			"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
-			"deprecated": "Glob versions prior to v9 are no longer supported",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"fs.realpath": "^1.0.0",
-				"inflight": "^1.0.4",
-				"inherits": "2",
-				"minimatch": "^3.1.1",
-				"once": "^1.3.0",
-				"path-is-absolute": "^1.0.0"
-			},
-			"engines": {
-				"node": "*"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/isaacs"
-			}
-		},
-		"node_modules/gluegun": {
-			"version": "5.2.0",
-			"resolved": "https://registry.npmjs.org/gluegun/-/gluegun-5.2.0.tgz",
-			"integrity": "sha512-jSUM5xUy2ztYFQANne17OUm/oAd7qSX7EBksS9bQDt9UvLPqcEkeWUebmaposb8Tx7eTTD8uJVWGRe6PYSsYkg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"apisauce": "^2.1.5",
-				"app-module-path": "^2.2.0",
-				"cli-table3": "0.6.0",
-				"colors": "1.4.0",
-				"cosmiconfig": "7.0.1",
-				"cross-spawn": "7.0.3",
-				"ejs": "3.1.8",
-				"enquirer": "2.3.6",
-				"execa": "5.1.1",
-				"fs-jetpack": "4.3.1",
-				"lodash.camelcase": "^4.3.0",
-				"lodash.kebabcase": "^4.1.1",
-				"lodash.lowercase": "^4.3.0",
-				"lodash.lowerfirst": "^4.3.1",
-				"lodash.pad": "^4.5.1",
-				"lodash.padend": "^4.6.1",
-				"lodash.padstart": "^4.6.1",
-				"lodash.repeat": "^4.1.0",
-				"lodash.snakecase": "^4.1.1",
-				"lodash.startcase": "^4.4.0",
-				"lodash.trim": "^4.5.1",
-				"lodash.trimend": "^4.5.1",
-				"lodash.trimstart": "^4.5.1",
-				"lodash.uppercase": "^4.3.0",
-				"lodash.upperfirst": "^4.3.1",
-				"ora": "4.0.2",
-				"pluralize": "^8.0.0",
-				"semver": "7.3.5",
-				"which": "2.0.2",
-				"yargs-parser": "^21.0.0"
-			},
-			"bin": {
-				"gluegun": "bin/gluegun"
-			}
-		},
-		"node_modules/gluegun/node_modules/ansi-regex": {
-			"version": "4.1.1",
-			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz",
-			"integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/gluegun/node_modules/ansi-styles": {
-			"version": "3.2.1",
-			"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-			"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"color-convert": "^1.9.0"
-			},
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/gluegun/node_modules/chalk": {
-			"version": "2.4.2",
-			"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-			"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-styles": "^3.2.1",
-				"escape-string-regexp": "^1.0.5",
-				"supports-color": "^5.3.0"
-			},
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/gluegun/node_modules/cli-cursor": {
-			"version": "3.1.0",
-			"resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz",
-			"integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"restore-cursor": "^3.1.0"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/gluegun/node_modules/color-convert": {
-			"version": "1.9.3",
-			"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-			"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"color-name": "1.1.3"
-			}
-		},
-		"node_modules/gluegun/node_modules/color-name": {
-			"version": "1.1.3",
-			"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-			"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/gluegun/node_modules/has-flag": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-			"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/gluegun/node_modules/is-interactive": {
-			"version": "1.0.0",
-			"resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz",
-			"integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/gluegun/node_modules/ora": {
-			"version": "4.0.2",
-			"resolved": "https://registry.npmjs.org/ora/-/ora-4.0.2.tgz",
-			"integrity": "sha512-YUOZbamht5mfLxPmk4M35CD/5DuOkAacxlEUbStVXpBAt4fyhBf+vZHI/HRkI++QUp3sNoeA2Gw4C+hi4eGSig==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"chalk": "^2.4.2",
-				"cli-cursor": "^3.1.0",
-				"cli-spinners": "^2.2.0",
-				"is-interactive": "^1.0.0",
-				"log-symbols": "^3.0.0",
-				"strip-ansi": "^5.2.0",
-				"wcwidth": "^1.0.1"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/gluegun/node_modules/restore-cursor": {
-			"version": "3.1.0",
-			"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz",
-			"integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"onetime": "^5.1.0",
-				"signal-exit": "^3.0.2"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/gluegun/node_modules/semver": {
-			"version": "7.3.5",
-			"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
-			"integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"lru-cache": "^6.0.0"
-			},
-			"bin": {
-				"semver": "bin/semver.js"
-			},
-			"engines": {
-				"node": ">=10"
-			}
-		},
-		"node_modules/gluegun/node_modules/strip-ansi": {
-			"version": "5.2.0",
-			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
-			"integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-regex": "^4.1.0"
-			},
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/gluegun/node_modules/supports-color": {
-			"version": "5.5.0",
-			"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-			"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"has-flag": "^3.0.0"
-			},
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/has-flag": {
-			"version": "4.0.0",
-			"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-			"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/http-proxy-agent": {
-			"version": "7.0.2",
-			"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz",
-			"integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"agent-base": "^7.1.0",
-				"debug": "^4.3.4"
-			},
-			"engines": {
-				"node": ">= 14"
-			}
-		},
-		"node_modules/https-proxy-agent": {
-			"version": "7.0.6",
-			"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz",
-			"integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"agent-base": "^7.1.2",
-				"debug": "4"
-			},
-			"engines": {
-				"node": ">= 14"
-			}
-		},
-		"node_modules/human-signals": {
-			"version": "2.1.0",
-			"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
-			"integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
-			"dev": true,
-			"license": "Apache-2.0",
-			"engines": {
-				"node": ">=10.17.0"
-			}
-		},
-		"node_modules/ieee754": {
-			"version": "1.2.1",
-			"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-			"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
-			"dev": true,
-			"funding": [
-				{
-					"type": "github",
-					"url": "https://github.com/sponsors/feross"
-				},
-				{
-					"type": "patreon",
-					"url": "https://www.patreon.com/feross"
-				},
-				{
-					"type": "consulting",
-					"url": "https://feross.org/support"
-				}
-			],
-			"license": "BSD-3-Clause"
-		},
-		"node_modules/immediate": {
-			"version": "3.0.6",
-			"resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
-			"integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/import-fresh": {
-			"version": "3.3.1",
-			"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz",
-			"integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"parent-module": "^1.0.0",
-				"resolve-from": "^4.0.0"
-			},
-			"engines": {
-				"node": ">=6"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/inflight": {
-			"version": "1.0.6",
-			"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-			"integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
-			"deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"once": "^1.3.0",
-				"wrappy": "1"
-			}
-		},
-		"node_modules/inherits": {
-			"version": "2.0.4",
-			"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
-			"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
-			"dev": true,
-			"license": "ISC"
-		},
-		"node_modules/is-arrayish": {
-			"version": "0.2.1",
-			"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
-			"integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/is-fullwidth-code-point": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
-			"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/is-interactive": {
-			"version": "2.0.0",
-			"resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz",
-			"integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=12"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/is-stream": {
-			"version": "2.0.1",
-			"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
-			"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/isarray": {
-			"version": "1.0.0",
-			"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-			"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/isexe": {
-			"version": "2.0.0",
-			"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-			"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
-			"dev": true,
-			"license": "ISC"
-		},
-		"node_modules/jake": {
-			"version": "10.9.2",
-			"resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz",
-			"integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==",
-			"dev": true,
-			"license": "Apache-2.0",
-			"dependencies": {
-				"async": "^3.2.3",
-				"chalk": "^4.0.2",
-				"filelist": "^1.0.4",
-				"minimatch": "^3.1.2"
-			},
-			"bin": {
-				"jake": "bin/cli.js"
-			},
-			"engines": {
-				"node": ">=10"
-			}
-		},
-		"node_modules/js-tokens": {
-			"version": "4.0.0",
-			"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
-			"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/json-parse-even-better-errors": {
-			"version": "2.3.1",
-			"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
-			"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/jszip": {
-			"version": "3.10.1",
-			"resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
-			"integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
-			"dev": true,
-			"license": "(MIT OR GPL-3.0-or-later)",
-			"dependencies": {
-				"lie": "~3.3.0",
-				"pako": "~1.0.2",
-				"readable-stream": "~2.3.6",
-				"setimmediate": "^1.0.5"
-			}
-		},
-		"node_modules/lie": {
-			"version": "3.3.0",
-			"resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
-			"integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"immediate": "~3.0.5"
-			}
-		},
-		"node_modules/lines-and-columns": {
-			"version": "1.2.4",
-			"resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
-			"integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.camelcase": {
-			"version": "4.3.0",
-			"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
-			"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.kebabcase": {
-			"version": "4.1.1",
-			"resolved": "https://registry.npmjs.org/lodash.kebabcase/-/lodash.kebabcase-4.1.1.tgz",
-			"integrity": "sha512-N8XRTIMMqqDgSy4VLKPnJ/+hpGZN+PHQiJnSenYqPaVV/NCqEogTnAdZLQiGKhxX+JCs8waWq2t1XHWKOmlY8g==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.lowercase": {
-			"version": "4.3.0",
-			"resolved": "https://registry.npmjs.org/lodash.lowercase/-/lodash.lowercase-4.3.0.tgz",
-			"integrity": "sha512-UcvP1IZYyDKyEL64mmrwoA1AbFu5ahojhTtkOUr1K9dbuxzS9ev8i4TxMMGCqRC9TE8uDaSoufNAXxRPNTseVA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.lowerfirst": {
-			"version": "4.3.1",
-			"resolved": "https://registry.npmjs.org/lodash.lowerfirst/-/lodash.lowerfirst-4.3.1.tgz",
-			"integrity": "sha512-UUKX7VhP1/JL54NXg2aq/E1Sfnjjes8fNYTNkPU8ZmsaVeBvPHKdbNaN79Re5XRL01u6wbq3j0cbYZj71Fcu5w==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.pad": {
-			"version": "4.5.1",
-			"resolved": "https://registry.npmjs.org/lodash.pad/-/lodash.pad-4.5.1.tgz",
-			"integrity": "sha512-mvUHifnLqM+03YNzeTBS1/Gr6JRFjd3rRx88FHWUvamVaT9k2O/kXha3yBSOwB9/DTQrSTLJNHvLBBt2FdX7Mg==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.padend": {
-			"version": "4.6.1",
-			"resolved": "https://registry.npmjs.org/lodash.padend/-/lodash.padend-4.6.1.tgz",
-			"integrity": "sha512-sOQs2aqGpbl27tmCS1QNZA09Uqp01ZzWfDUoD+xzTii0E7dSQfRKcRetFwa+uXaxaqL+TKm7CgD2JdKP7aZBSw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.padstart": {
-			"version": "4.6.1",
-			"resolved": "https://registry.npmjs.org/lodash.padstart/-/lodash.padstart-4.6.1.tgz",
-			"integrity": "sha512-sW73O6S8+Tg66eY56DBk85aQzzUJDtpoXFBgELMd5P/SotAguo+1kYO6RuYgXxA4HJH3LFTFPASX6ET6bjfriw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.repeat": {
-			"version": "4.1.0",
-			"resolved": "https://registry.npmjs.org/lodash.repeat/-/lodash.repeat-4.1.0.tgz",
-			"integrity": "sha512-eWsgQW89IewS95ZOcr15HHCX6FVDxq3f2PNUIng3fyzsPev9imFQxIYdFZ6crl8L56UR6ZlGDLcEb3RZsCSSqw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.snakecase": {
-			"version": "4.1.1",
-			"resolved": "https://registry.npmjs.org/lodash.snakecase/-/lodash.snakecase-4.1.1.tgz",
-			"integrity": "sha512-QZ1d4xoBHYUeuouhEq3lk3Uq7ldgyFXGBhg04+oRLnIz8o9T65Eh+8YdroUwn846zchkA9yDsDl5CVVaV2nqYw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.startcase": {
-			"version": "4.4.0",
-			"resolved": "https://registry.npmjs.org/lodash.startcase/-/lodash.startcase-4.4.0.tgz",
-			"integrity": "sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.trim": {
-			"version": "4.5.1",
-			"resolved": "https://registry.npmjs.org/lodash.trim/-/lodash.trim-4.5.1.tgz",
-			"integrity": "sha512-nJAlRl/K+eiOehWKDzoBVrSMhK0K3A3YQsUNXHQa5yIrKBAhsZgSu3KoAFoFT+mEgiyBHddZ0pRk1ITpIp90Wg==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.trimend": {
-			"version": "4.5.1",
-			"resolved": "https://registry.npmjs.org/lodash.trimend/-/lodash.trimend-4.5.1.tgz",
-			"integrity": "sha512-lsD+k73XztDsMBKPKvzHXRKFNMohTjoTKIIo4ADLn5dA65LZ1BqlAvSXhR2rPEC3BgAUQnzMnorqDtqn2z4IHA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.trimstart": {
-			"version": "4.5.1",
-			"resolved": "https://registry.npmjs.org/lodash.trimstart/-/lodash.trimstart-4.5.1.tgz",
-			"integrity": "sha512-b/+D6La8tU76L/61/aN0jULWHkT0EeJCmVstPBn/K9MtD2qBW83AsBNrr63dKuWYwVMO7ucv13QNO/Ek/2RKaQ==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.uppercase": {
-			"version": "4.3.0",
-			"resolved": "https://registry.npmjs.org/lodash.uppercase/-/lodash.uppercase-4.3.0.tgz",
-			"integrity": "sha512-+Nbnxkj7s8K5U8z6KnEYPGUOGp3woZbB7Ecs7v3LkkjLQSm2kP9SKIILitN1ktn2mB/tmM9oSlku06I+/lH7QA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/lodash.upperfirst": {
-			"version": "4.3.1",
-			"resolved": "https://registry.npmjs.org/lodash.upperfirst/-/lodash.upperfirst-4.3.1.tgz",
-			"integrity": "sha512-sReKOYJIJf74dhJONhU4e0/shzi1trVbSWDOhKYE5XV2O+H7Sb2Dihwuc7xWxVl+DgFPyTqIN3zMfT9cq5iWDg==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/log-symbols": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-3.0.0.tgz",
-			"integrity": "sha512-dSkNGuI7iG3mfvDzUuYZyvk5dD9ocYCYzNU6CYDE6+Xqd+gwme6Z00NS3dUh8mq/73HaEtT7m6W+yUPtU6BZnQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"chalk": "^2.4.2"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/log-symbols/node_modules/ansi-styles": {
-			"version": "3.2.1",
-			"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-			"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"color-convert": "^1.9.0"
-			},
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/log-symbols/node_modules/chalk": {
-			"version": "2.4.2",
-			"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-			"integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-styles": "^3.2.1",
-				"escape-string-regexp": "^1.0.5",
-				"supports-color": "^5.3.0"
-			},
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/log-symbols/node_modules/color-convert": {
-			"version": "1.9.3",
-			"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-			"integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"color-name": "1.1.3"
-			}
-		},
-		"node_modules/log-symbols/node_modules/color-name": {
-			"version": "1.1.3",
-			"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-			"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/log-symbols/node_modules/has-flag": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-			"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/log-symbols/node_modules/supports-color": {
-			"version": "5.5.0",
-			"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-			"integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"has-flag": "^3.0.0"
-			},
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/lru-cache": {
-			"version": "6.0.0",
-			"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
-			"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"yallist": "^4.0.0"
-			},
-			"engines": {
-				"node": ">=10"
-			}
-		},
-		"node_modules/merge-stream": {
-			"version": "2.0.0",
-			"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
-			"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/mimic-fn": {
-			"version": "2.1.0",
-			"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
-			"integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/minimatch": {
-			"version": "3.1.2",
-			"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
-			"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"brace-expansion": "^1.1.7"
-			},
-			"engines": {
-				"node": "*"
-			}
-		},
-		"node_modules/ms": {
-			"version": "2.1.3",
-			"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
-			"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/npm-run-path": {
-			"version": "4.0.1",
-			"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
-			"integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"path-key": "^3.0.0"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/object-assign": {
-			"version": "4.1.1",
-			"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-			"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=0.10.0"
-			}
-		},
-		"node_modules/once": {
-			"version": "1.4.0",
-			"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-			"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"wrappy": "1"
-			}
-		},
-		"node_modules/onetime": {
-			"version": "5.1.2",
-			"resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
-			"integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"mimic-fn": "^2.1.0"
-			},
-			"engines": {
-				"node": ">=6"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/ora": {
-			"version": "7.0.1",
-			"resolved": "https://registry.npmjs.org/ora/-/ora-7.0.1.tgz",
-			"integrity": "sha512-0TUxTiFJWv+JnjWm4o9yvuskpEJLXTcng8MJuKd+SzAzp2o+OP3HWqNhB4OdJRt1Vsd9/mR0oyaEYlOnL7XIRw==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"chalk": "^5.3.0",
-				"cli-cursor": "^4.0.0",
-				"cli-spinners": "^2.9.0",
-				"is-interactive": "^2.0.0",
-				"is-unicode-supported": "^1.3.0",
-				"log-symbols": "^5.1.0",
-				"stdin-discarder": "^0.1.0",
-				"string-width": "^6.1.0",
-				"strip-ansi": "^7.1.0"
-			},
-			"engines": {
-				"node": ">=16"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/ora/node_modules/chalk": {
-			"version": "5.4.1",
-			"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz",
-			"integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": "^12.17.0 || ^14.13 || >=16.0.0"
-			},
-			"funding": {
-				"url": "https://github.com/chalk/chalk?sponsor=1"
-			}
-		},
-		"node_modules/ora/node_modules/emoji-regex": {
-			"version": "10.4.0",
-			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.4.0.tgz",
-			"integrity": "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/ora/node_modules/is-unicode-supported": {
-			"version": "1.3.0",
-			"resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz",
-			"integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=12"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/ora/node_modules/log-symbols": {
-			"version": "5.1.0",
-			"resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-5.1.0.tgz",
-			"integrity": "sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"chalk": "^5.0.0",
-				"is-unicode-supported": "^1.1.0"
-			},
-			"engines": {
-				"node": ">=12"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/ora/node_modules/string-width": {
-			"version": "6.1.0",
-			"resolved": "https://registry.npmjs.org/string-width/-/string-width-6.1.0.tgz",
-			"integrity": "sha512-k01swCJAgQmuADB0YIc+7TuatfNvTBVOoaUWJjTB9R4VJzR5vNWzf5t42ESVZFPS8xTySF7CAdV4t/aaIm3UnQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"eastasianwidth": "^0.2.0",
-				"emoji-regex": "^10.2.1",
-				"strip-ansi": "^7.0.1"
-			},
-			"engines": {
-				"node": ">=16"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/pako": {
-			"version": "1.0.11",
-			"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
-			"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
-			"dev": true,
-			"license": "(MIT AND Zlib)"
-		},
-		"node_modules/parent-module": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
-			"integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"callsites": "^3.0.0"
-			},
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/parse-json": {
-			"version": "5.2.0",
-			"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz",
-			"integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"@babel/code-frame": "^7.0.0",
-				"error-ex": "^1.3.1",
-				"json-parse-even-better-errors": "^2.3.0",
-				"lines-and-columns": "^1.1.6"
-			},
-			"engines": {
-				"node": ">=8"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/path-is-absolute": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-			"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=0.10.0"
-			}
-		},
-		"node_modules/path-key": {
-			"version": "3.1.1",
-			"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
-			"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/path-type": {
-			"version": "4.0.0",
-			"resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
-			"integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/picocolors": {
-			"version": "1.1.1",
-			"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
-			"integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==",
-			"dev": true,
-			"license": "ISC"
-		},
-		"node_modules/pluralize": {
-			"version": "8.0.0",
-			"resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz",
-			"integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/process-nextick-args": {
-			"version": "2.0.1",
-			"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-			"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/readable-stream": {
-			"version": "2.3.8",
-			"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
-			"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"core-util-is": "~1.0.0",
-				"inherits": "~2.0.3",
-				"isarray": "~1.0.0",
-				"process-nextick-args": "~2.0.0",
-				"safe-buffer": "~5.1.1",
-				"string_decoder": "~1.1.1",
-				"util-deprecate": "~1.0.1"
-			}
-		},
-		"node_modules/require-directory": {
-			"version": "2.1.1",
-			"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
-			"integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=0.10.0"
-			}
-		},
-		"node_modules/resolve-from": {
-			"version": "4.0.0",
-			"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
-			"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=4"
-			}
-		},
-		"node_modules/resolve-pkg-maps": {
-			"version": "1.0.0",
-			"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
-			"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
-			"dev": true,
-			"license": "MIT",
-			"funding": {
-				"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
-			}
-		},
-		"node_modules/restore-cursor": {
-			"version": "4.0.0",
-			"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz",
-			"integrity": "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"onetime": "^5.1.0",
-				"signal-exit": "^3.0.2"
-			},
-			"engines": {
-				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/rimraf": {
-			"version": "2.7.1",
-			"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
-			"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
-			"deprecated": "Rimraf versions prior to v4 are no longer supported",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"glob": "^7.1.3"
-			},
-			"bin": {
-				"rimraf": "bin.js"
-			}
-		},
-		"node_modules/safe-buffer": {
-			"version": "5.1.2",
-			"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-			"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/semver": {
-			"version": "7.7.1",
-			"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
-			"integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
-			"dev": true,
-			"license": "ISC",
-			"bin": {
-				"semver": "bin/semver.js"
-			},
-			"engines": {
-				"node": ">=10"
-			}
-		},
-		"node_modules/setimmediate": {
-			"version": "1.0.5",
-			"resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
-			"integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/shebang-command": {
-			"version": "2.0.0",
-			"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
-			"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"shebang-regex": "^3.0.0"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/shebang-regex": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
-			"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/signal-exit": {
-			"version": "3.0.7",
-			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
-			"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
-			"dev": true,
-			"license": "ISC"
-		},
-		"node_modules/stdin-discarder": {
-			"version": "0.1.0",
-			"resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.1.0.tgz",
-			"integrity": "sha512-xhV7w8S+bUwlPTb4bAOUQhv8/cSS5offJuX8GQGq32ONF0ZtDWKfkdomM3HMRA+LhX6um/FZ0COqlwsjD53LeQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"bl": "^5.0.0"
-			},
-			"engines": {
-				"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-			},
-			"funding": {
-				"url": "https://github.com/sponsors/sindresorhus"
-			}
-		},
-		"node_modules/string_decoder": {
-			"version": "1.1.1",
-			"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-			"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"safe-buffer": "~5.1.0"
-			}
-		},
-		"node_modules/string-width": {
-			"version": "4.2.3",
-			"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
-			"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"emoji-regex": "^8.0.0",
-				"is-fullwidth-code-point": "^3.0.0",
-				"strip-ansi": "^6.0.1"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/string-width/node_modules/ansi-regex": {
-			"version": "5.0.1",
-			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-			"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/string-width/node_modules/strip-ansi": {
-			"version": "6.0.1",
-			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-			"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-regex": "^5.0.1"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/strip-ansi": {
-			"version": "7.1.0",
-			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
-			"integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-regex": "^6.0.1"
-			},
-			"engines": {
-				"node": ">=12"
-			},
-			"funding": {
-				"url": "https://github.com/chalk/strip-ansi?sponsor=1"
-			}
-		},
-		"node_modules/strip-final-newline": {
-			"version": "2.0.0",
-			"resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
-			"integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=6"
-			}
-		},
-		"node_modules/supports-color": {
-			"version": "7.2.0",
-			"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-			"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"has-flag": "^4.0.0"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/tsx": {
-			"version": "4.19.3",
-			"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.3.tgz",
-			"integrity": "sha512-4H8vUNGNjQ4V2EOoGw005+c+dGuPSnhpPBPHBtsZdGZBk/iJb4kguGlPWaZTZ3q5nMtFOEsY0nRDlh9PJyd6SQ==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"esbuild": "~0.25.0",
-				"get-tsconfig": "^4.7.5"
-			},
-			"bin": {
-				"tsx": "dist/cli.mjs"
-			},
-			"engines": {
-				"node": ">=18.0.0"
-			},
-			"optionalDependencies": {
-				"fsevents": "~2.3.3"
-			}
-		},
-		"node_modules/typescript": {
-			"version": "5.8.2",
-			"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.2.tgz",
-			"integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==",
-			"dev": true,
-			"license": "Apache-2.0",
-			"bin": {
-				"tsc": "bin/tsc",
-				"tsserver": "bin/tsserver"
-			},
-			"engines": {
-				"node": ">=14.17"
-			}
-		},
-		"node_modules/util-deprecate": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-			"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
-			"dev": true,
-			"license": "MIT"
-		},
-		"node_modules/wcwidth": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
-			"integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"defaults": "^1.0.3"
-			}
-		},
-		"node_modules/which": {
-			"version": "2.0.2",
-			"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
-			"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
-			"dev": true,
-			"license": "ISC",
-			"dependencies": {
-				"isexe": "^2.0.0"
-			},
-			"bin": {
-				"node-which": "bin/node-which"
-			},
-			"engines": {
-				"node": ">= 8"
-			}
-		},
-		"node_modules/wrap-ansi": {
-			"version": "7.0.0",
-			"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
-			"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-styles": "^4.0.0",
-				"string-width": "^4.1.0",
-				"strip-ansi": "^6.0.0"
-			},
-			"engines": {
-				"node": ">=10"
-			},
-			"funding": {
-				"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
-			}
-		},
-		"node_modules/wrap-ansi/node_modules/ansi-regex": {
-			"version": "5.0.1",
-			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
-			"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/wrap-ansi/node_modules/strip-ansi": {
-			"version": "6.0.1",
-			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
-			"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"ansi-regex": "^5.0.1"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
-		"node_modules/wrappy": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-			"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
-			"dev": true,
-			"license": "ISC"
-		},
-		"node_modules/y18n": {
-			"version": "5.0.8",
-			"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
-			"integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
-			"dev": true,
-			"license": "ISC",
-			"engines": {
-				"node": ">=10"
-			}
-		},
-		"node_modules/yallist": {
-			"version": "4.0.0",
-			"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-			"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
-			"dev": true,
-			"license": "ISC"
-		},
-		"node_modules/yaml": {
-			"version": "1.10.2",
-			"resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz",
-			"integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==",
-			"dev": true,
-			"license": "ISC",
-			"engines": {
-				"node": ">= 6"
-			}
-		},
-		"node_modules/yargs": {
-			"version": "17.7.2",
-			"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
-			"integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"cliui": "^8.0.1",
-				"escalade": "^3.1.1",
-				"get-caller-file": "^2.0.5",
-				"require-directory": "^2.1.1",
-				"string-width": "^4.2.3",
-				"y18n": "^5.0.5",
-				"yargs-parser": "^21.1.1"
-			},
-			"engines": {
-				"node": ">=12"
-			}
-		},
-		"node_modules/yargs-parser": {
-			"version": "21.1.1",
-			"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
-			"integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
-			"dev": true,
-			"license": "ISC",
-			"engines": {
-				"node": ">=12"
-			}
-		}
-	}
-}

benchmark/package.json

@@ -1,30 +0,0 @@
-{
-	"name": "benchmark",
-	"version": "0.1.0",
-	"private": true,
-	"main": "out/run.js",
-	"scripts": {
-		"build": "npm run compile && cd .. && npm run compile && npm run build:webview",
-		"lint": "eslint src --ext ts",
-		"check-types": "tsc --noEmit",
-		"compile": "rm -rf out && tsc -p tsconfig.json",
-		"cli": "npm run compile && npx dotenvx run -f .env.local -- tsx src/cli.ts",
-		"clean": "rimraf out",
-		"clean:exercises": "cd exercises && git checkout -f && git clean -fd",
-		"docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..",
-		"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
-		"docker:start": "npm run docker:build && npm run docker:run",
-		"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/bash",
-		"docker:cli": "docker exec -it -w /home/vscode/repo/benchmark $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) xvfb-run npm run cli --",
-		"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q)",
-		"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q)",
-		"docker:clean": "npm run docker:stop && npm run docker:rm"
-	},
-	"devDependencies": {
-		"@vscode/test-electron": "^2.4.0",
-		"gluegun": "^5.1.2",
-		"tsx": "^4.19.3",
-		"typescript": "^5.4.5",
-		"yargs": "^17.7.2"
-	}
-}

benchmark/prompts/cpp.md

@@ -1,17 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, you can compile your code and run the tests with:
-
-```
-mkdir -p build && cd build
-cmake -G "Unix Makefiles" -DEXERCISM_RUN_ALL_TESTS=1 ..
-make
-```
-
-Note that running `make` will compile the tests and generate compile time errors. Once the errors are fixed, running `make` will build and run the tests.
-
-Do not alter the test file; it should be run as-is.

benchmark/prompts/go.md

@@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `go test`. Do not alter the test file; it should be run as-is.

benchmark/prompts/java.md

@@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `./gradlew test`. Do not alter the test file; it should be run as-is.

benchmark/prompts/javascript.md

@@ -1,9 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `pnpm test`. Do not alter the test file; it should be run as-is.
-
-Before running the tests make sure your environment is set up by running `pnpm install` to install the dependencies.

benchmark/prompts/python.md

@@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `uv run python3 -m pytest -o markers=task [name]_test.py`. Do not alter the test file; it should be run as-is.

benchmark/prompts/rust.md

@@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `cargo test`. Do not alter the test file; it should be run as-is.

benchmark/src/cli.ts

@@ -1,171 +0,0 @@
-import * as fs from "fs"
-import * as path from "path"
-
-import { build, filesystem, GluegunPrompt } from "gluegun"
-import { runTests } from "@vscode/test-electron"
-
-// console.log(__dirname)
-// <...>/Roo-Code/benchmark/src
-
-const extensionDevelopmentPath = path.resolve(__dirname, "../../")
-const extensionTestsPath = path.resolve(__dirname, "../out/runExercise")
-const promptsPath = path.resolve(__dirname, "../prompts")
-const exercisesPath = path.resolve(__dirname, "../../../exercises")
-const languages = ["cpp", "go", "java", "javascript", "python", "rust"]
-
-async function runAll({ runId, model }: { runId: number; model: string }) {
-	for (const language of languages) {
-		await runLanguage({ runId, model, language })
-	}
-}
-
-async function runLanguage({ runId, model, language }: { runId: number; model: string; language: string }) {
-	const languagePath = path.resolve(exercisesPath, language)
-
-	if (!fs.existsSync(languagePath)) {
-		console.error(`Language directory ${languagePath} does not exist`)
-		process.exit(1)
-	}
-
-	const exercises = filesystem
-		.subdirectories(languagePath)
-		.map((exercise) => path.basename(exercise))
-		.filter((exercise) => !exercise.startsWith("."))
-
-	for (const exercise of exercises) {
-		await runExercise({ runId, model, language, exercise })
-	}
-}
-
-async function runExercise({
-	runId,
-	model,
-	language,
-	exercise,
-}: {
-	runId: number
-	model: string
-	language: string
-	exercise: string
-}) {
-	const workspacePath = path.resolve(exercisesPath, language, exercise)
-	const promptPath = path.resolve(promptsPath, `${language}.md`)
-
-	const extensionTestsEnv = {
-		PROMPT_PATH: promptPath,
-		WORKSPACE_PATH: workspacePath,
-		OPENROUTER_MODEL_ID: model,
-		RUN_ID: runId.toString(),
-	}
-
-	if (fs.existsSync(path.resolve(workspacePath, "usage.json"))) {
-		console.log(`Test result exists for ${language} / ${exercise}, skipping`)
-		return
-	}
-
-	console.log(`Running ${language} / ${exercise}`)
-
-	await runTests({
-		extensionDevelopmentPath,
-		extensionTestsPath,
-		launchArgs: [workspacePath, "--disable-extensions"],
-		extensionTestsEnv,
-	})
-}
-
-async function askLanguage(prompt: GluegunPrompt) {
-	const languages = filesystem.subdirectories(exercisesPath)
-
-	if (languages.length === 0) {
-		throw new Error(`No languages found in ${exercisesPath}`)
-	}
-
-	const { language } = await prompt.ask<{ language: string }>({
-		type: "select",
-		name: "language",
-		message: "Which language?",
-		choices: languages.map((language) => path.basename(language)).filter((language) => !language.startsWith(".")),
-	})
-
-	return language
-}
-
-async function askExercise(prompt: GluegunPrompt, language: string) {
-	const exercises = filesystem.subdirectories(path.join(exercisesPath, language))
-
-	if (exercises.length === 0) {
-		throw new Error(`No exercises found for ${language}`)
-	}
-
-	const { exercise } = await prompt.ask<{ exercise: string }>({
-		type: "select",
-		name: "exercise",
-		message: "Which exercise?",
-		choices: exercises.map((exercise) => path.basename(exercise)),
-	})
-
-	return exercise
-}
-
-async function createRun({ model }: { model: string }): Promise<{ id: number; model: string }> {
-	const response = await fetch("http://localhost:3000/api/runs", {
-		method: "POST",
-		body: JSON.stringify({ model }),
-	})
-
-	if (!response.ok) {
-		throw new Error(`Failed to create run: ${response.statusText}`)
-	}
-
-	const {
-		run: [run],
-	} = await response.json()
-	return run
-}
-
-async function main() {
-	const cli = build()
-		.brand("benchmark-runner")
-		.src(__dirname)
-		.help()
-		.version()
-		.command({
-			name: "run",
-			run: ({ config, parameters }) => {
-				config.language = parameters.first
-				config.exercise = parameters.second
-
-				if (parameters.options["runId"]) {
-					config.runId = parameters.options["runId"]
-				}
-			},
-		})
-		.defaultCommand() // Use the default command if no args.
-		.create()
-
-	const { print, prompt, config } = await cli.run(process.argv)
-
-	try {
-		const model = "anthropic/claude-3.7-sonnet"
-		const runId = config.runId ? Number(config.runId) : (await createRun({ model })).id
-
-		if (config.language === "all") {
-			console.log("Running all exercises for all languages")
-			await runAll({ runId, model })
-		} else if (config.exercise === "all") {
-			console.log(`Running all exercises for ${config.language}`)
-			await runLanguage({ runId, model, language: config.language })
-		} else {
-			const language = config.language || (await askLanguage(prompt))
-			const exercise = config.exercise || (await askExercise(prompt, language))
-			await runExercise({ runId, model, language, exercise })
-		}
-
-		process.exit(0)
-	} catch (error) {
-		print.error(error)
-		process.exit(1)
-	}
-}
-
-main()

benchmark/src/runExercise.ts

@@ -1,94 +0,0 @@
-import * as fs from "fs/promises"
-import * as path from "path"
-
-import * as vscode from "vscode"
-
-import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
-
-import { waitUntilReady, waitUntilCompleted, sleep } from "./utils"
-
-export async function run() {
-	/**
-	 * Validate environment variables.
-	 */
-
-	const runId = process.env.RUN_ID
-	const openRouterApiKey = process.env.OPENROUTER_API_KEY
-	const openRouterModelId = process.env.OPENROUTER_MODEL_ID
-	const promptPath = process.env.PROMPT_PATH
-	const workspacePath = process.env.WORKSPACE_PATH
-
-	if (!runId || !openRouterApiKey || !openRouterModelId || !promptPath || !workspacePath) {
-		throw new Error("ENV not configured.")
-	}
-
-	const prompt = await fs.readFile(promptPath, "utf-8")
-
-	/**
-	 * Activate the extension.
-	 */
-
-	const extension = vscode.extensions.getExtension<RooCodeAPI>("RooVeterinaryInc.roo-cline")
-
-	if (!extension) {
-		throw new Error("Extension not found.")
-	}
-
-	const api = extension.isActive ? extension.exports : await extension.activate()
-
-	/**
-	 * Wait for the Roo Code to be ready to accept tasks.
-	 */
-
-	await waitUntilReady({ api })
-
-	/**
-	 * Configure Roo Code as needed.
-	 *
-	 * Use Claude 3.7 Sonnet via OpenRouter.
-	 * Don't require approval for anything.
-	 * Run any command without approval.
-	 * Disable checkpoints (for performance).
-	 */
-
-	await api.setConfiguration({
-		apiProvider: "openrouter",
-		openRouterApiKey,
-		openRouterModelId,
-		autoApprovalEnabled: true,
-		alwaysAllowReadOnly: true,
-		alwaysAllowWrite: true,
-		alwaysAllowExecute: true,
-		alwaysAllowBrowser: true,
-		alwaysApproveResubmit: true,
-		alwaysAllowMcp: true,
-		alwaysAllowModeSwitch: true,
-		enableCheckpoints: false,
-	})
-
-	await vscode.workspace
-		.getConfiguration("roo-cline")
-		.update("allowedCommands", ["*"], vscode.ConfigurationTarget.Global)
-
-	await sleep(2_000)
-
-	/**
-	 * Run the task and wait up to 10 minutes for it to complete.
-	 */
-
-	const startTime = Date.now()
-	const taskId = await api.startNewTask(prompt)
-
-	let usage: TokenUsage | undefined = undefined
-
-	try {
-		usage = await waitUntilCompleted({ api, taskId, timeout: 5 * 60 * 1_000 }) // 5m
-	} catch (e) {
-		usage = api.getTokenUsage(taskId)
-	}
-
-	if (usage) {
-		const content = JSON.stringify({ runId: parseInt(runId), ...usage, duration: Date.now() - startTime }, null, 2)
-		await fs.writeFile(path.resolve(workspacePath, "usage.json"), content)
-	}
-}

benchmark/src/utils.ts

@@ -1,111 +0,0 @@
-import * as vscode from "vscode"
-
-import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
-
-type WaitForOptions = {
-	timeout?: number
-	interval?: number
-}
-
-export const waitFor = (
-	condition: (() => Promise<boolean>) | (() => boolean),
-	{ timeout = 30_000, interval = 250 }: WaitForOptions = {},
-) => {
-	let timeoutId: NodeJS.Timeout | undefined = undefined
-
-	return Promise.race([
-		new Promise<void>((resolve) => {
-			const check = async () => {
-				const result = condition()
-				const isSatisfied = result instanceof Promise ? await result : result
-
-				if (isSatisfied) {
-					if (timeoutId) {
-						clearTimeout(timeoutId)
-						timeoutId = undefined
-					}
-
-					resolve()
-				} else {
-					setTimeout(check, interval)
-				}
-			}
-
-			check()
-		}),
-		new Promise((_, reject) => {
-			timeoutId = setTimeout(() => {
-				reject(new Error(`Timeout after ${Math.floor(timeout / 1000)}s`))
-			}, timeout)
-		}),
-	])
-}
-
-type WaitUntilReadyOptions = WaitForOptions & {
-	api: RooCodeAPI
-}
-
-export const waitUntilReady = async ({ api, ...options }: WaitUntilReadyOptions) => {
-	await vscode.commands.executeCommand("roo-cline.SidebarProvider.focus")
-	await waitFor(() => api.isReady(), options)
-}
-
-type WaitUntilAbortedOptions = WaitForOptions & {
-	api: RooCodeAPI
-	taskId: string
-}
-
-export const waitUntilAborted = async ({ api, taskId, ...options }: WaitUntilAbortedOptions) => {
-	const set = new Set<string>()
-	api.on("taskAborted", (taskId) => set.add(taskId))
-	await waitFor(() => set.has(taskId), options)
-}
-
-type WaitUntilCompletedOptions = WaitForOptions & {
-	api: RooCodeAPI
-	taskId: string
-}
-
-export const waitUntilCompleted = async ({ api, taskId, ...options }: WaitUntilCompletedOptions) => {
-	const map = new Map<string, TokenUsage>()
-	api.on("taskCompleted", (taskId, usage) => map.set(taskId, usage))
-	await waitFor(() => map.has(taskId), options)
-	return map.get(taskId)
-}
-
-export const waitForCompletion = async ({
-	api,
-	taskId,
-	...options
-}: WaitUntilReadyOptions & {
-	taskId: string
-}) => waitFor(() => !!getCompletion({ api, taskId }), options)
-
-export const getCompletion = ({ api, taskId }: { api: RooCodeAPI; taskId: string }) =>
-	api.getMessages(taskId).find(({ say, partial }) => say === "completion_result" && partial === false)
-
-type WaitForMessageOptions = WaitUntilReadyOptions & {
-	taskId: string
-	include: string
-	exclude?: string
-}
-
-export const waitForMessage = async ({ api, taskId, include, exclude, ...options }: WaitForMessageOptions) =>
-	waitFor(() => !!getMessage({ api, taskId, include, exclude }), options)
-
-type GetMessageOptions = {
-	api: RooCodeAPI
-	taskId: string
-	include: string
-	exclude?: string
-}
-
-export const getMessage = ({ api, taskId, include, exclude }: GetMessageOptions) =>
-	api
-		.getMessages(taskId)
-		.find(
-			({ type, text }) =>
-				type === "say" && text && text.includes(include) && (!exclude || !text.includes(exclude)),
-		)
-
-export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

evals/.env.sample

@@ -0,0 +1 @@
+BENCHMARKS_DB_PATH=file:/tmp/evals.db

evals/.gitignore

@@ -0,0 +1,42 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# Dependencies
+node_modules
+.pnp
+.pnp.js
+
+# Local env files
+.env
+.env.*
+!.env.sample
+
+# Testing
+coverage
+
+# Turbo
+.turbo
+
+# Vercel
+.vercel
+
+# Next.js
+next-env.d.ts
+
+# Build Outputs
+.next/
+out/
+build
+dist
+*.tsbuildinfo
+
+# Debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Misc
+.DS_Store
+*.pem
+
+# Evals
+evals

evals/.npmrc

@@ -0,0 +1,2 @@
+# https://github.com/vercel/next.js/issues/68805
+public-hoist-pattern[]=*libsql*

evals/.tool-versions

@@ -0,0 +1,4 @@
+nodejs v20.18.1
+python 3.13.2
+golang 1.24.2
+rust 1.85.1

evals/README.md

@@ -0,0 +1,21 @@
+# Run Roo Code Evals
+
+## Get Started
+
+NOTE: This is MacOS only for now!
+
+Clone the Roo Code repo:
+
+```sh
+git clone https://github.com/RooVetGit/Roo-Code.git
+cd Roo-Code
+```
+
+Run the setup script:
+
+```sh
+cd evals
+./scripts/setup.sh
+```
+
+Navigate to [localhost:3000](http://localhost:3000/) in your browser.

evals/apps/cli/eslint.config.mjs

@@ -0,0 +1,4 @@
+import { config } from "@evals/eslint-config/base"
+
+/** @type {import("eslint").Linter.Config} */
+export default [...config]

evals/apps/cli/package.json

@@ -0,0 +1,25 @@
+{
+	"name": "@evals/cli",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"lint": "eslint src --ext ts --max-warnings=0",
+		"check-types": "tsc --noEmit",
+		"format": "prettier --write src",
+		"dev": "dotenvx run -f ../../.env -- tsx src/index.ts"
+	},
+	"dependencies": {
+		"@evals/db": "workspace:^",
+		"@evals/ipc": "workspace:^",
+		"@evals/lib": "workspace:^",
+		"@evals/types": "workspace:^",
+		"execa": "^9.5.2",
+		"gluegun": "^5.1.2",
+		"p-map": "^7.0.3",
+		"p-wait-for": "^5.0.2"
+	},
+	"devDependencies": {
+		"@evals/eslint-config": "workspace:^",
+		"@evals/typescript-config": "workspace:^"
+	}
+}

evals/apps/cli/src/exercises.ts

@@ -0,0 +1,31 @@
+import * as path from "path"
+import * as fs from "fs"
+
+import { filesystem } from "gluegun"
+
+import { type ExerciseLanguage, exerciseLanguages } from "@evals/types"
+
+import { exercisesPath } from "./paths.js"
+
+let exercisesByLanguage: Record<ExerciseLanguage, string[]> | null = null
+
+export const getExercises = () => {
+	if (exercisesByLanguage !== null) {
+		return exercisesByLanguage
+	}
+
+	const getLanguageExercises = (language: ExerciseLanguage) =>
+		fs.existsSync(path.resolve(exercisesPath, language))
+			? filesystem
+					.subdirectories(path.resolve(exercisesPath, language))
+					.map((exercise) => path.basename(exercise))
+					.filter((exercise) => !exercise.startsWith("."))
+			: []
+
+	exercisesByLanguage = exerciseLanguages.reduce(
+		(collect, language) => ({ ...collect, [language]: getLanguageExercises(language) }),
+		{} as Record<ExerciseLanguage, string[]>,
+	)
+
+	return exercisesByLanguage
+}

evals/apps/cli/src/index.ts

@@ -0,0 +1,473 @@
+import * as fs from "fs"
+import * as path from "path"
+import * as os from "os"
+
+import pMap from "p-map"
+import pWaitFor from "p-wait-for"
+import { execa, parseCommandString } from "execa"
+import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
+
+import {
+	type ExerciseLanguage,
+	exerciseLanguages,
+	RooCodeEventName,
+	IpcOrigin,
+	IpcMessageType,
+	TaskCommandName,
+	rooCodeDefaults,
+} from "@evals/types"
+import {
+	type Run,
+	findRun,
+	createRun,
+	finishRun,
+	type Task,
+	createTask,
+	getTasks,
+	updateTask,
+	createTaskMetrics,
+	updateTaskMetrics,
+} from "@evals/db"
+import { IpcServer, IpcClient } from "@evals/ipc"
+
+import { __dirname, extensionDevelopmentPath, exercisesPath } from "./paths.js"
+import { getExercises } from "./exercises.js"
+
+const maxConcurrency = 2
+const taskTimeLimit = 5 * 60 * 1_000
+
+const testCommands: Record<ExerciseLanguage, { commands: string[]; timeout?: number; cwd?: string }> = {
+	go: { commands: ["go test"] }, // timeout 15s bash -c "cd '$dir' && go test > /dev/null 2>&1"
+	java: { commands: ["./gradlew test"] }, // timeout --foreground 15s bash -c "cd '$dir' && ./gradlew test > /dev/null 2>&1"
+	javascript: { commands: ["pnpm install", "pnpm test"], timeout: 30_000 }, // timeout 30s bash -c "cd '$dir' && pnpm install >/dev/null 2>&1 && pnpm test >/dev/null 2>&1"
+	python: { commands: ["uv run python3 -m pytest -o markers=task *_test.py"] }, // timeout 15s bash -c "cd '$dir' && uv run python3 -m pytest -o markers=task *_test.py"
+	rust: { commands: ["cargo test"] }, // timeout 15s bash -c "cd '$dir' && cargo test > /dev/null 2>&1"
+}
+
+const run = async (toolbox: GluegunToolbox) => {
+	const { config, prompt } = toolbox
+
+	let { language, exercise } = config
+
+	if (![undefined, ...exerciseLanguages, "all"].includes(language)) {
+		throw new Error(`Language is invalid: ${language}`)
+	}
+
+	if (!["undefined", "string"].includes(typeof exercise)) {
+		throw new Error(`Exercise is invalid: ${exercise}`)
+	}
+
+	const id = config.runId ? Number(config.runId) : undefined
+	let run: Run
+
+	if (id) {
+		run = await findRun(id)
+	} else {
+		run = await createRun({
+			model: rooCodeDefaults.openRouterModelId!,
+			pid: process.pid,
+			socketPath: path.resolve(os.tmpdir(), `roo-code-evals-${crypto.randomUUID()}.sock`),
+		})
+
+		if (language === "all") {
+			for (const language of exerciseLanguages) {
+				const exercises = getExercises()[language as ExerciseLanguage]
+
+				await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), {
+					concurrency: 10,
+				})
+			}
+		} else if (exercise === "all") {
+			const exercises = getExercises()[language as ExerciseLanguage]
+			await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), { concurrency: 10 })
+		} else {
+			language = language || (await askLanguage(prompt))
+			exercise = exercise || (await askExercise(prompt, language))
+			await createTask({ runId: run.id, language, exercise })
+		}
+	}
+
+	const tasks = await getTasks(run.id)
+
+	if (!tasks[0]) {
+		throw new Error("No tasks found.")
+	}
+
+	console.log(await execa({ cwd: exercisesPath })`git config user.name "Roo Code"`)
+	console.log(await execa({ cwd: exercisesPath })`git config user.email "[email protected]"`)
+	console.log(await execa({ cwd: exercisesPath })`git checkout -f`)
+	console.log(await execa({ cwd: exercisesPath })`git clean -fd`)
+	console.log(await execa({ cwd: exercisesPath })`git checkout -b runs/${run.id} main`)
+
+	fs.writeFileSync(
+		path.resolve(exercisesPath, "settings.json"),
+		JSON.stringify({ ...rooCodeDefaults, ...run.settings }, null, 2),
+	)
+
+	const server = new IpcServer(run.socketPath, () => {})
+	server.listen()
+
+	// server.on(IpcMessageType.Connect, (clientId) => {
+	// 	server.send(clientId, {
+	// 		type: IpcMessageType.TaskEvent,
+	// 		origin: IpcOrigin.Server,
+	// 		data: { eventName: RooCodeEventName.Connect, taskId: -1 },
+	// 	})
+	// })
+
+	const runningPromises: Promise<void>[] = []
+
+	const processTask = async (task: Task) => {
+		if (task.finishedAt === null) {
+			await runExercise({ run, task, server })
+		}
+
+		if (task.passed === null) {
+			const passed = await runUnitTest({ task })
+			await updateTask(task.id, { passed })
+		}
+	}
+
+	for (const task of tasks) {
+		const taskPromise = processTask(task)
+		runningPromises.push(taskPromise)
+
+		taskPromise.finally(() => {
+			const index = runningPromises.indexOf(taskPromise)
+
+			if (index > -1) {
+				runningPromises.splice(index, 1)
+			}
+		})
+
+		if (runningPromises.length >= maxConcurrency) {
+			await Promise.race(runningPromises)
+		}
+	}
+
+	await Promise.all(runningPromises)
+
+	const result = await finishRun(run.id)
+	try {
+		console.log("[cli#run]", result)
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	} catch (error) {
+		// console.error(error)
+	}
+
+	console.log(await execa({ cwd: exercisesPath })`git add .`)
+	console.log(await execa({ cwd: exercisesPath })`git commit -m ${`Run #${run.id}`} --no-verify`)
+}
+
+const runExercise = async ({ run, task, server }: { run: Run; task: Task; server: IpcServer }) => {
+	const { language, exercise } = task
+	const prompt = fs.readFileSync(path.resolve(exercisesPath, `prompts/${language}.md`), "utf-8")
+	const dirname = path.dirname(run.socketPath)
+	const taskSocketPath = path.resolve(dirname, `${dirname}/task-${task.id}.sock`)
+
+	const controller = new AbortController()
+	const cancelSignal = controller.signal
+
+	// If debugging:
+	// Use --wait --log trace or --verbose.
+	const codeCommand = `code --disable-workspace-trust`
+
+	await execa({
+		env: {
+			ROO_CODE_IPC_SOCKET_PATH: taskSocketPath,
+		},
+		shell: "/bin/bash",
+		cancelSignal,
+	})`${codeCommand} -n ${path.resolve(exercisesPath, language, exercise)}`
+
+	// If debugging:
+	// Don't await execa and store result as subprocess.
+	// subprocess.stdout.pipe(process.stdout)
+
+	// Give VSCode some time to spawn before connectint to its unix socket.
+	await new Promise((resolve) => setTimeout(resolve, 1_000))
+	console.log(`Connecting to ${taskSocketPath}`)
+
+	const createClient = (taskSocketPath: string) => {
+		const ipcClient = new IpcClient(taskSocketPath)
+
+		ipcClient.on(IpcMessageType.Ack, (ack) => {
+			console.log(`[cli#runExercise | ${language} / ${exercise}] ack`, ack)
+		})
+
+		return ipcClient
+	}
+
+	let tries = 0
+	let client = createClient(taskSocketPath)
+
+	while (++tries < 5) {
+		try {
+			await pWaitFor(() => client.isReady, { interval: 100, timeout: 5_000 })
+			break
+		} catch (error) {
+			console.error(error)
+			client.disconnect()
+			client = createClient(taskSocketPath)
+		}
+	}
+
+	let isTaskFinished = false
+	let isClientDisconnected = false
+
+	client.on(IpcMessageType.Disconnect, async () => {
+		console.log(`[cli#runExercise | ${language} / ${exercise}] disconnect`)
+		isTaskFinished = true
+		isClientDisconnected = true
+	})
+
+	const ignoreEvents: RooCodeEventName[] = [
+		// RooCodeEventName.Message,
+		RooCodeEventName.TaskTokenUsageUpdated,
+		RooCodeEventName.TaskAskResponded,
+	]
+
+	let taskStartedAt = Date.now()
+	let taskMetricsId: number | undefined
+	let rooTaskId: string | undefined
+
+	client.on(IpcMessageType.TaskEvent, async (taskEvent) => {
+		const { eventName, payload } = taskEvent
+
+		server.broadcast({
+			type: IpcMessageType.TaskEvent,
+			origin: IpcOrigin.Server,
+			relayClientId: client.clientId!,
+			data: { ...taskEvent, taskId: task.id },
+		})
+
+		if (!ignoreEvents.includes(eventName)) {
+			console.log(`[cli#runExercise | ${language} / ${exercise}] taskEvent -> ${eventName}`)
+			console.log(payload)
+		}
+
+		if (eventName === RooCodeEventName.TaskStarted) {
+			taskStartedAt = Date.now()
+
+			const taskMetrics = await createTaskMetrics({
+				cost: 0,
+				tokensIn: 0,
+				tokensOut: 0,
+				tokensContext: 0,
+				duration: 0,
+				cacheWrites: 0,
+				cacheReads: 0,
+			})
+
+			await updateTask(task.id, { taskMetricsId: taskMetrics.id, startedAt: new Date() })
+
+			taskStartedAt = Date.now()
+			taskMetricsId = taskMetrics.id
+			rooTaskId = payload[0]
+		}
+
+		if (
+			(eventName === RooCodeEventName.TaskTokenUsageUpdated || eventName === RooCodeEventName.TaskCompleted) &&
+			taskMetricsId
+		) {
+			const duration = Date.now() - taskStartedAt
+
+			const { totalCost, totalTokensIn, totalTokensOut, contextTokens, totalCacheWrites, totalCacheReads } =
+				payload[1]
+
+			await updateTaskMetrics(taskMetricsId, {
+				cost: totalCost,
+				tokensIn: totalTokensIn,
+				tokensOut: totalTokensOut,
+				tokensContext: contextTokens,
+				duration,
+				cacheWrites: totalCacheWrites ?? 0,
+				cacheReads: totalCacheReads ?? 0,
+			})
+		}
+
+		if (eventName === RooCodeEventName.TaskCompleted || eventName === RooCodeEventName.TaskAborted) {
+			await updateTask(task.id, { finishedAt: new Date() })
+			isTaskFinished = true
+		}
+	})
+
+	if (client.isReady) {
+		client.sendMessage({
+			type: IpcMessageType.TaskCommand,
+			origin: IpcOrigin.Client,
+			clientId: client.clientId!,
+			data: {
+				commandName: TaskCommandName.StartNewTask,
+				data: {
+					configuration: {
+						...rooCodeDefaults,
+						openRouterApiKey: process.env.OPENROUTER_API_KEY!,
+						...run.settings,
+					},
+					text: prompt,
+					newTab: true,
+				},
+			},
+		})
+
+		console.log(`[cli#runExercise | ${language} / ${exercise}] starting task`)
+	} else {
+		console.log(`[cli#runExercise | ${language} / ${exercise}] unable to connect`)
+		client.disconnect()
+		isTaskFinished = true
+		isClientDisconnected = true
+	}
+
+	try {
+		await pWaitFor(() => isTaskFinished, { interval: 1_000, timeout: taskTimeLimit })
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	} catch (error) {
+		console.log(`[cli#runExercise | ${language} / ${exercise}] time limit reached`)
+
+		if (rooTaskId && !isClientDisconnected) {
+			client.sendMessage({
+				type: IpcMessageType.TaskCommand,
+				origin: IpcOrigin.Client,
+				clientId: client.clientId!,
+				data: { commandName: TaskCommandName.CancelTask, data: rooTaskId },
+			})
+
+			await new Promise((resolve) => setTimeout(resolve, 2_000))
+		}
+
+		await updateTask(task.id, { finishedAt: new Date() })
+	}
+
+	if (!isClientDisconnected) {
+		try {
+			if (rooTaskId) {
+				client.sendMessage({
+					type: IpcMessageType.TaskCommand,
+					origin: IpcOrigin.Client,
+					clientId: client.clientId!,
+					data: { commandName: TaskCommandName.CloseTask, data: rooTaskId },
+				})
+			}
+
+			client.disconnect()
+		} catch (error) {
+			console.error(error)
+		}
+	}
+
+	// try {
+	// 	console.log(`[cli#runExercise | ${language} / ${exercise}] aborting subprocess`)
+	// 	controller.abort()
+	// 	await subprocess
+	// } catch (error) {
+	// }
+}
+
+const runUnitTest = async ({ task }: { task: Task }) => {
+	const cmd = testCommands[task.language]
+	const exercisePath = path.resolve(exercisesPath, task.language, task.exercise)
+	const cwd = cmd.cwd ? path.resolve(exercisePath, cmd.cwd) : exercisePath
+	const commands = cmd.commands.map((cs) => parseCommandString(cs))
+
+	let passed = true
+
+	for (const command of commands) {
+		// const controller = new AbortController()
+		// const cancelSignal = controller.signal
+		// const timeout = setTimeout(() => controller.abort(), cmd.timeout ?? 15_000)
+
+		try {
+			const result = await execa({ cwd, shell: true, reject: false /* , cancelSignal */ })`${command}`
+			// console.log('[cli#run] execa result =', { ...result, cwd, command })
+
+			// clearTimeout(timeout)
+
+			if (result.failed) {
+				passed = false
+				break
+			}
+		} catch (error) {
+			console.log("[cli#run] execa error =", error)
+			passed = false
+			break
+		}
+	}
+
+	return passed
+}
+
+const askLanguage = async (prompt: GluegunPrompt) => {
+	const { language } = await prompt.ask<{ language: ExerciseLanguage }>({
+		type: "select",
+		name: "language",
+		message: "Which language?",
+		choices: [...exerciseLanguages],
+	})
+
+	return language
+}
+
+const askExercise = async (prompt: GluegunPrompt, language: ExerciseLanguage) => {
+	const exercises = filesystem.subdirectories(path.join(exercisesPath, language))
+
+	if (exercises.length === 0) {
+		throw new Error(`No exercises found for ${language}`)
+	}
+
+	const { exercise } = await prompt.ask<{ exercise: string }>({
+		type: "select",
+		name: "exercise",
+		message: "Which exercise?",
+		choices: exercises.map((exercise) => path.basename(exercise)).filter((exercise) => !exercise.startsWith(".")),
+	})
+
+	return exercise
+}
+
+const main = async () => {
+	const cli = build()
+		.brand("cli")
+		.src(__dirname)
+		.help()
+		.version()
+		.command({
+			name: "run",
+			description: "Run an eval",
+			run: ({ config, parameters }) => {
+				config.language = parameters.first
+				config.exercise = parameters.second
+
+				if (parameters.options["runId"]) {
+					config.runId = parameters.options["runId"]
+				}
+			},
+		})
+		.defaultCommand()
+		.create()
+
+	const toolbox = await cli.run(process.argv)
+	const { command } = toolbox
+
+	switch (command?.name) {
+		case "run":
+			await run(toolbox)
+			break
+	}
+
+	process.exit(0)
+}
+
+if (!fs.existsSync(extensionDevelopmentPath)) {
+	console.error(`"extensionDevelopmentPath" does not exist.`)
+	process.exit(1)
+}
+
+if (!fs.existsSync(exercisesPath)) {
+	console.error(
+		`Exercises path does not exist. Please run "git clone https://github.com/cte/Roo-Code-Benchmark.git exercises".`,
+	)
+	process.exit(1)
+}
+
+main()

evals/apps/cli/src/paths.ts

@@ -0,0 +1,7 @@
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+export const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..")
+export const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "evals")

evals/apps/cli/tsconfig.json

@@ -0,0 +1,5 @@
+{
+	"extends": "@evals/typescript-config/base.json",
+	"include": ["src"],
+	"exclude": ["node_modules"]
+}

evals/apps/web/components.json

@@ -0,0 +1,21 @@
+{
+	"$schema": "https://ui.shadcn.com/schema.json",
+	"style": "new-york",
+	"rsc": true,
+	"tsx": true,
+	"tailwind": {
+		"config": "",
+		"css": "src/app/globals.css",
+		"baseColor": "neutral",
+		"cssVariables": true,
+		"prefix": ""
+	},
+	"aliases": {
+		"components": "@/components",
+		"utils": "@/lib/utils",
+		"ui": "@/components/ui",
+		"lib": "@/lib",
+		"hooks": "@/hooks"
+	},
+	"iconLibrary": "lucide"
+}

evals/apps/web/eslint.config.mjs

@@ -0,0 +1,17 @@
+import { nextJsConfig } from "@evals/eslint-config/next-js"
+
+/** @type {import("eslint").Linter.Config} */
+export default [
+	...nextJsConfig,
+	{
+		rules: {
+			"no-unused-vars": "off",
+			"@typescript-eslint/no-unused-vars": [
+				"error",
+				{
+					caughtErrorsIgnorePattern: "^_",
+				},
+			],
+		},
+	},
+]

evals/apps/web/next.config.ts

@@ -0,0 +1,7 @@
+import type { NextConfig } from "next"
+
+const nextConfig: NextConfig = {
+	/* config options here */
+}
+
+export default nextConfig

evals/apps/web/package.json

@@ -0,0 +1,55 @@
+{
+	"name": "@evals/web",
+	"private": true,
+	"scripts": {
+		"lint": "next lint",
+		"check-types": "tsc -b",
+		"dev": "dotenvx run -f ../../.env -- next dev --turbopack",
+		"format": "prettier --write src",
+		"build": "next build",
+		"start": "next start"
+	},
+	"dependencies": {
+		"@evals/db": "workspace:^",
+		"@evals/ipc": "workspace:^",
+		"@evals/types": "workspace:^",
+		"@hookform/resolvers": "^4.1.3",
+		"@radix-ui/react-dialog": "^1.1.6",
+		"@radix-ui/react-label": "^2.1.2",
+		"@radix-ui/react-popover": "^1.1.6",
+		"@radix-ui/react-scroll-area": "^1.2.3",
+		"@radix-ui/react-select": "^2.1.6",
+		"@radix-ui/react-separator": "^1.1.2",
+		"@radix-ui/react-slot": "^1.1.2",
+		"@radix-ui/react-tabs": "^1.1.3",
+		"@radix-ui/react-tooltip": "^1.1.8",
+		"@tanstack/react-query": "^5.69.0",
+		"class-variance-authority": "^0.7.1",
+		"clsx": "^2.1.1",
+		"cmdk": "^1.1.0",
+		"fuzzysort": "^3.1.0",
+		"lucide-react": "^0.479.0",
+		"next": "15.2.2",
+		"next-themes": "^0.4.6",
+		"p-map": "^7.0.3",
+		"ps-tree": "^1.2.0",
+		"react": "^19.0.0",
+		"react-dom": "^19.0.0",
+		"react-hook-form": "^7.54.2",
+		"react-use": "^17.6.0",
+		"sonner": "^2.0.2",
+		"tailwind-merge": "^3.0.2",
+		"tailwindcss-animate": "^1.0.7",
+		"vaul": "^1.1.2",
+		"zod": "^3.24.2"
+	},
+	"devDependencies": {
+		"@evals/eslint-config": "workspace:^",
+		"@evals/typescript-config": "workspace:^",
+		"@tailwindcss/postcss": "^4",
+		"@types/ps-tree": "^1.1.6",
+		"@types/react": "^19",
+		"@types/react-dom": "^19",
+		"tailwindcss": "^4"
+	}
+}

evals/apps/web/postcss.config.mjs

@@ -0,0 +1,5 @@
+const config = {
+	plugins: ["@tailwindcss/postcss"],
+}
+
+export default config

evals/apps/web/src/app/api/runs/[id]/stream/route.ts

@@ -0,0 +1,39 @@
+import type { NextRequest } from "next/server"
+
+import { findRun } from "@evals/db"
+import { IpcMessageType } from "@evals/types"
+import { IpcClient } from "@evals/ipc"
+
+import { SSEStream } from "@/lib/server/sse-stream"
+
+export const dynamic = "force-dynamic"
+
+export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
+	const { id } = await params
+	const requestId = crypto.randomUUID()
+	const stream = new SSEStream()
+	const run = await findRun(Number(id))
+	const client = new IpcClient(run.socketPath, () => {})
+
+	const write = async (data: string | object) => {
+		// console.log(`[stream#${requestId}] write`, data)
+		const success = await stream.write(data)
+
+		if (!success) {
+			client.disconnect()
+		}
+	}
+
+	console.log(`[stream#${requestId}] connect`)
+	client.on(IpcMessageType.Connect, () => write("connect"))
+	client.on(IpcMessageType.Disconnect, () => write("disconnect"))
+	client.on(IpcMessageType.TaskEvent, write)
+
+	request.signal.addEventListener("abort", () => {
+		console.log(`[stream#${requestId}] abort`)
+		client.disconnect()
+		stream.close().catch(() => {})
+	})
+
+	return stream.getResponse()
+}

evals/apps/web/src/app/api/runs/route.ts

@@ -0,0 +1,12 @@
+import { NextResponse } from "next/server"
+
+import { createRun } from "@evals/db"
+
+export async function POST(request: Request) {
+	try {
+		const run = await createRun(await request.json())
+		return NextResponse.json({ run }, { status: 201 })
+	} catch (error) {
+		return NextResponse.json({ error: (error as Error).message }, { status: 500 })
+	}
+}

evals/apps/web/src/app/api/tasks/route.ts

@@ -0,0 +1,12 @@
+import { NextResponse } from "next/server"
+
+import { createTask } from "@evals/db"
+
+export async function POST(request: Request) {
+	try {
+		const task = await createTask(await request.json())
+		return NextResponse.json({ task }, { status: 201 })
+	} catch (error) {
+		return NextResponse.json({ error: (error as Error).message }, { status: 500 })
+	}
+}

evals/apps/web/src/app/globals.css

@@ -0,0 +1,141 @@
+@import "tailwindcss";
+
+@plugin "tailwindcss-animate";
+
+@custom-variant dark (&:is(.dark *));
+
+:root {
+	--radius: 0.625rem;
+	--background: oklch(1 0 0);
+	--foreground: oklch(0.145 0 0);
+	--card: oklch(1 0 0);
+	--card-foreground: oklch(0.145 0 0);
+	--popover: oklch(1 0 0);
+	--popover-foreground: oklch(0.145 0 0);
+	--primary: oklch(0.205 0 0);
+	--primary-foreground: oklch(0.985 0 0);
+	--secondary: oklch(0.97 0 0);
+	--secondary-foreground: oklch(0.205 0 0);
+	--muted: oklch(0.97 0 0);
+	--muted-foreground: oklch(0.556 0 0);
+	--accent: oklch(0.97 0 0);
+	--accent-foreground: oklch(0.205 0 0);
+	--destructive: oklch(0.577 0.245 27.325);
+	--border: oklch(0.922 0 0);
+	--input: oklch(0.922 0 0);
+	--ring: oklch(0.708 0 0);
+	--chart-1: oklch(0.646 0.222 41.116);
+	--chart-2: oklch(0.6 0.118 184.704);
+	--chart-3: oklch(0.398 0.07 227.392);
+	--chart-4: oklch(0.828 0.189 84.429);
+	--chart-5: oklch(0.769 0.188 70.08);
+	--sidebar: oklch(0.985 0 0);
+	--sidebar-foreground: oklch(0.145 0 0);
+	--sidebar-primary: oklch(0.205 0 0);
+	--sidebar-primary-foreground: oklch(0.985 0 0);
+	--sidebar-accent: oklch(0.97 0 0);
+	--sidebar-accent-foreground: oklch(0.205 0 0);
+	--sidebar-border: oklch(0.922 0 0);
+	--sidebar-ring: oklch(0.708 0 0);
+}
+
+.dark {
+	--background: oklch(23.66% 0.0198 271.79);
+	--foreground: oklch(75.15% 0.0477 278.41);
+	--card: oklch(0.205 0 0);
+	--card-foreground: oklch(0.985 0 0);
+	--popover: var(--primary);
+	--popover-foreground: oklch(0.985 0 0);
+	--primary: oklch(29.33% 0.0295 276.18);
+	--primary-foreground: var(--accent);
+	--secondary: var(--primary);
+	--secondary-foreground: var(--foreground);
+	--muted: oklch(28.27% 0.0207 273.06);
+	--muted-foreground: oklch(75.15% 0.0477 278.41 / 75%);
+	--accent: oklch(70.21% 0.1813 328.71);
+	--accent-foreground: oklch(1 0 0 / 75%);
+	--destructive: oklch(72.14% 0.1616 15.49);
+	--border: var(--primary);
+	--input: var(--primary);
+	--ring: oklch(83.63% 0.1259 176.52);
+	--chart-1: oklch(0.488 0.243 264.376);
+	--chart-2: oklch(0.696 0.17 162.48);
+	--chart-3: oklch(0.769 0.188 70.08);
+	--chart-4: oklch(0.627 0.265 303.9);
+	--chart-5: oklch(0.645 0.246 16.439);
+	--sidebar: oklch(0.205 0 0);
+	--sidebar-foreground: oklch(0.985 0 0);
+	--sidebar-primary: oklch(0.488 0.243 264.376);
+	--sidebar-primary-foreground: oklch(0.985 0 0);
+	--sidebar-accent: oklch(0.269 0 0);
+	--sidebar-accent-foreground: oklch(0.985 0 0);
+	--sidebar-border: oklch(1 0 0 / 10%);
+	--sidebar-ring: oklch(0.556 0 0);
+}
+
+@theme inline {
+	--color-background: var(--background);
+	--color-foreground: var(--foreground);
+	--color-sidebar-ring: var(--sidebar-ring);
+	--color-sidebar-border: var(--sidebar-border);
+	--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+	--color-sidebar-accent: var(--sidebar-accent);
+	--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+	--color-sidebar-primary: var(--sidebar-primary);
+	--color-sidebar-foreground: var(--sidebar-foreground);
+	--color-sidebar: var(--sidebar);
+	--color-chart-5: var(--chart-5);
+	--color-chart-4: var(--chart-4);
+	--color-chart-3: var(--chart-3);
+	--color-chart-2: var(--chart-2);
+	--color-chart-1: var(--chart-1);
+	--color-ring: var(--ring);
+	--color-input: var(--input);
+	--color-border: var(--border);
+	--color-destructive: var(--destructive);
+	--color-accent-foreground: var(--accent-foreground);
+	--color-accent: var(--accent);
+	--color-muted-foreground: var(--muted-foreground);
+	--color-muted: var(--muted);
+	--color-secondary-foreground: var(--secondary-foreground);
+	--color-secondary: var(--secondary);
+	--color-primary-foreground: var(--primary-foreground);
+	--color-primary: var(--primary);
+	--color-popover-foreground: var(--popover-foreground);
+	--color-popover: var(--popover);
+	--color-card-foreground: var(--card-foreground);
+	--color-card: var(--card);
+	--radius-sm: calc(var(--radius) - 4px);
+	--radius-md: calc(var(--radius) - 2px);
+	--radius-lg: var(--radius);
+	--radius-xl: calc(var(--radius) + 4px);
+
+	--animate-hop: hop 0.8s ease-in-out infinite;
+
+	@keyframes hop {
+		0%,
+		100% {
+			transform: none;
+			animation-timing-function: cubic-bezier(0.8, 0, 1, 1);
+		}
+		50% {
+			transform: translateY(-8px);
+			animation-timing-function: cubic-bezier(0, 0, 0.2, 1);
+		}
+	}
+}
+
+@layer base {
+	* {
+		@apply border-border outline-ring/50;
+	}
+	html,
+	body {
+		height: 100%;
+	}
+	body {
+		@apply bg-background text-foreground;
+		scrollbar-color: rgba(0, 0, 0, 0.2) transparent; /* Firefox */
+		scrollbar-width: thin;
+	}
+}

evals/apps/web/src/app/home.tsx

@@ -0,0 +1,72 @@
+"use client"
+
+import { useRouter } from "next/navigation"
+import { Rocket } from "lucide-react"
+
+import type { Run, TaskMetrics } from "@evals/db"
+
+import { formatCurrency, formatDuration } from "@/lib"
+import { Button, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
+import { useMemo } from "react"
+import Link from "next/link"
+
+export function Home({ runs }: { runs: (Run & { taskMetrics: TaskMetrics | null })[] }) {
+	const router = useRouter()
+
+	const visibleRuns = useMemo(() => runs.filter((run) => run.taskMetrics !== null), [runs])
+
+	return (
+		<>
+			<Table className="border border-t-0">
+				<TableHeader>
+					<TableRow>
+						<TableHead>ID</TableHead>
+						<TableHead>Model</TableHead>
+						<TableHead>Timestamp</TableHead>
+						<TableHead>Passed</TableHead>
+						<TableHead>Failed</TableHead>
+						<TableHead>% Correct</TableHead>
+						<TableHead>Cost</TableHead>
+						<TableHead>Duration</TableHead>
+					</TableRow>
+				</TableHeader>
+				<TableBody>
+					{visibleRuns.length ? (
+						visibleRuns.map(({ taskMetrics, ...run }) => (
+							<TableRow key={run.id}>
+								<TableCell>
+									<Button variant="link" asChild>
+										<Link href={`/runs/${run.id}`}>{run.id}</Link>
+									</Button>
+								</TableCell>
+								<TableCell>{run.model}</TableCell>
+								<TableCell>{new Date(run.createdAt).toLocaleString()}</TableCell>
+								<TableCell>{run.passed}</TableCell>
+								<TableCell>{run.failed}</TableCell>
+								<TableCell>{((run.passed / (run.passed + run.failed)) * 100).toFixed(1)}%</TableCell>
+								<TableCell>{formatCurrency(taskMetrics!.cost)}</TableCell>
+								<TableCell>{formatDuration(taskMetrics!.duration)}</TableCell>
+							</TableRow>
+						))
+					) : (
+						<TableRow>
+							<TableCell colSpan={8} className="text-center">
+								No eval runs yet.
+								<Button variant="link" onClick={() => router.push("/runs/new")}>
+									Launch
+								</Button>
+								one now.
+							</TableCell>
+						</TableRow>
+					)}
+				</TableBody>
+			</Table>
+			<Button
+				variant="default"
+				className="absolute top-4 right-12 size-12 rounded-full"
+				onClick={() => router.push("/runs/new")}>
+				<Rocket className="size-6" />
+			</Button>
+		</>
+	)
+}

evals/apps/web/src/app/layout.tsx

@@ -0,0 +1,35 @@
+import type { Metadata } from "next"
+import { Geist, Geist_Mono } from "next/font/google"
+
+import { ThemeProvider, ReactQueryProvider } from "@/components/providers"
+import { Toaster } from "@/components/ui"
+import { Header } from "@/components/layout/header"
+
+import "./globals.css"
+
+const fontSans = Geist({ variable: "--font-sans", subsets: ["latin"] })
+const fontMono = Geist_Mono({ variable: "--font-mono", subsets: ["latin"] })
+
+export const metadata: Metadata = {
+	title: "Roo Code Benchmarks",
+}
+
+export default function RootLayout({
+	children,
+}: Readonly<{
+	children: React.ReactNode
+}>) {
+	return (
+		<html lang="en">
+			<body className={`${fontSans.variable} ${fontMono.variable} font-sans antialiased pb-12`}>
+				<ThemeProvider attribute="class" defaultTheme="system" enableSystem disableTransitionOnChange>
+					<ReactQueryProvider>
+						<Header />
+						{children}
+					</ReactQueryProvider>
+				</ThemeProvider>
+				<Toaster />
+			</body>
+		</html>
+	)
+}

evals/apps/web/src/app/page.tsx

@@ -0,0 +1,10 @@
+import { getRuns } from "@evals/db"
+
+import { Home } from "./home"
+
+export const dynamic = "force-dynamic"
+
+export default async function Page() {
+	const runs = await getRuns()
+	return <Home runs={runs} />
+}

evals/apps/web/src/app/runs/[id]/connection-status.tsx

@@ -0,0 +1,69 @@
+"use client"
+
+import { useCallback } from "react"
+import { Skull } from "lucide-react"
+
+import { killProcessTree } from "@/lib/server/processes"
+import { EventSourceStatus } from "@/hooks/use-event-source"
+import { useProcessList } from "@/hooks/use-process-tree"
+import { cn } from "@/lib/utils"
+import { Button } from "@/components/ui"
+
+type ConnectionStatusProps = {
+	status: EventSourceStatus
+	pid: number | null
+}
+
+export const ConnectionStatus = (connectionStatus: ConnectionStatusProps) => {
+	const { data: pids, isLoading } = useProcessList(connectionStatus.pid)
+	const status = isLoading ? "loading" : pids === null ? "dead" : connectionStatus.status
+
+	const onKill = useCallback(async () => {
+		if (connectionStatus.pid) {
+			await killProcessTree(connectionStatus.pid)
+			window.location.reload()
+		}
+	}, [connectionStatus.pid])
+
+	return (
+		<div>
+			<div className="flex items-center gap-2">
+				<div className="flex items-center gap-2">
+					<div>Status:</div>
+					<div className="capitalize">{status}</div>
+				</div>
+				<div className="relative">
+					<div
+						className={cn("absolute size-2.5 rounded-full opacity-50 animate-ping", {
+							"bg-gray-500": status === "loading",
+							"bg-green-500": status === "connected",
+							"bg-amber-500": status === "waiting",
+							"bg-rose-500": status === "error" || status === "dead",
+						})}
+					/>
+					<div
+						className={cn("size-2.5 rounded-full", {
+							"bg-gray-500": status === "loading",
+							"bg-green-500": status === "connected",
+							"bg-amber-500": status === "waiting",
+							"bg-rose-500": status === "error" || status === "dead",
+						})}
+					/>
+				</div>
+			</div>
+			<div className="flex items-center gap-2">
+				<div>PIDs:</div>
+				<div className="font-mono text-sm">{connectionStatus.pid}</div>
+				{status === "connected" && (
+					<>
+						<div className="font-mono text-sm text-muted-foreground">{pids?.join(" ")}</div>
+						<Button variant="ghost" size="sm" onClick={onKill}>
+							Kill
+							<Skull />
+						</Button>
+					</>
+				)}
+			</div>
+		</div>
+	)
+}

evals/apps/web/src/app/runs/[id]/page.tsx

@@ -0,0 +1,14 @@
+import { findRun } from "@evals/db"
+
+import { Run } from "./run"
+
+export default async function Page({ params }: { params: Promise<{ id: string }> }) {
+	const { id } = await params
+	const run = await findRun(Number(id))
+
+	return (
+		<div className="max-w-3xl mx-auto px-12 p-12">
+			<Run run={run} />
+		</div>
+	)
+}

evals/apps/web/src/app/runs/[id]/run.tsx

@@ -0,0 +1,140 @@
+"use client"
+
+import { useState, useRef, useEffect } from "react"
+import { LoaderCircle, SquareTerminal } from "lucide-react"
+
+import * as db from "@evals/db"
+
+import { formatCurrency, formatDuration, formatTokens } from "@/lib"
+import { useRunStatus } from "@/hooks/use-run-status"
+import {
+	Drawer,
+	DrawerContent,
+	DrawerHeader,
+	DrawerTitle,
+	ScrollArea,
+	Separator,
+	Table,
+	TableBody,
+	TableCell,
+	TableHead,
+	TableHeader,
+	TableRow,
+} from "@/components/ui"
+
+import { TaskStatus } from "./task-status"
+import { ConnectionStatus } from "./connection-status"
+
+export function Run({ run }: { run: db.Run }) {
+	const { tasks, status, output, outputCounts } = useRunStatus(run)
+	const scrollAreaRef = useRef<HTMLDivElement>(null)
+	const [selectedTask, setSelectedTask] = useState<db.Task>()
+
+	useEffect(() => {
+		if (selectedTask) {
+			const scrollArea = scrollAreaRef.current
+
+			if (scrollArea) {
+				scrollArea.scrollTo({
+					top: scrollArea.scrollHeight,
+					behavior: "smooth",
+				})
+			}
+		}
+	}, [selectedTask, outputCounts])
+
+	return (
+		<>
+			<div>
+				<div className="mb-2">
+					<div>
+						<div>{run.model}</div>
+						{run.description && <div className="text-sm text-muted-foreground">{run.description}</div>}
+					</div>
+					<ConnectionStatus status={status} pid={run.pid} />
+				</div>
+				{!tasks ? (
+					<LoaderCircle className="size-4 animate-spin" />
+				) : (
+					<Table className="border">
+						<TableHeader>
+							<TableRow>
+								<TableHead>Exercise</TableHead>
+								<TableHead className="text-center">Tokens In / Out</TableHead>
+								<TableHead>Context</TableHead>
+								<TableHead>Duration</TableHead>
+								<TableHead>Cost</TableHead>
+							</TableRow>
+						</TableHeader>
+						<TableBody>
+							{tasks.map((task) => (
+								<TableRow key={task.id}>
+									<TableCell>
+										<div className="flex items-center gap-2">
+											<TaskStatus task={task} />
+											<div>
+												{task.language}/{task.exercise}
+											</div>
+											{(outputCounts[task.id] ?? 0) > 0 && (
+												<div
+													className="flex items-center gap-1 cursor-pointer"
+													onClick={() => setSelectedTask(task)}>
+													<SquareTerminal className="size-4" />
+													<div className="font-mono text-xs text-foreground/50">
+														{outputCounts[task.id]}
+													</div>
+												</div>
+											)}
+										</div>
+									</TableCell>
+									{task.taskMetrics ? (
+										<>
+											<TableCell className="font-mono text-xs">
+												<div className="flex items-center justify-evenly">
+													<div>{formatTokens(task.taskMetrics.tokensIn)}</div>/
+													<div>{formatTokens(task.taskMetrics.tokensOut)}</div>
+												</div>
+											</TableCell>
+											<TableCell className="font-mono text-xs">
+												{formatTokens(task.taskMetrics.tokensContext)}
+											</TableCell>
+											<TableCell className="font-mono text-xs">
+												{formatDuration(task.taskMetrics.duration)}
+											</TableCell>
+											<TableCell className="font-mono text-xs">
+												{formatCurrency(task.taskMetrics.cost)}
+											</TableCell>
+										</>
+									) : (
+										<TableCell colSpan={4} />
+									)}
+								</TableRow>
+							))}
+						</TableBody>
+					</Table>
+				)}
+			</div>
+			<Drawer open={!!selectedTask} onOpenChange={() => setSelectedTask(undefined)}>
+				<DrawerContent>
+					<div className="mx-auto w-full max-w-2xl">
+						<DrawerHeader>
+							<DrawerTitle>
+								{selectedTask?.language}/{selectedTask?.exercise}
+							</DrawerTitle>
+						</DrawerHeader>
+						<div className="font-mono text-xs pb-12">
+							{selectedTask && (
+								<ScrollArea viewportRef={scrollAreaRef} className="h-96 rounded-sm border">
+									<div className="p-4">
+										<h4 className="mb-4 text-sm font-medium leading-none">Tags</h4>
+										{output.get(selectedTask.id)?.map((line, i) => <div key={i}>{line}</div>)}
+									</div>
+								</ScrollArea>
+							)}
+						</div>
+					</div>
+				</DrawerContent>
+			</Drawer>
+		</>
+	)
+}

evals/apps/web/src/app/runs/[id]/task-status.tsx

@@ -0,0 +1,21 @@
+import { CircleCheck, CircleDashed, CircleSlash, LoaderCircle } from "lucide-react"
+
+import { type Task } from "@evals/db"
+
+type TaskStatusProps = {
+	task: Task
+}
+
+export const TaskStatus = ({ task }: TaskStatusProps) => {
+	return task.passed === false ? (
+		<CircleSlash className="size-4 text-destructive" />
+	) : task.passed === true ? (
+		<CircleCheck className="size-4 text-green-500" />
+	) : task.startedAt ? (
+		<LoaderCircle className="size-4 animate-spin" />
+	) : task.finishedAt ? (
+		<LoaderCircle className="size-4 animate-spin" />
+	) : (
+		<CircleDashed className="size-4" />
+	)
+}

evals/apps/web/src/app/runs/new/new-run.tsx

@@ -0,0 +1,313 @@
+"use client"
+
+import { useCallback, useRef, useState } from "react"
+import { useRouter } from "next/navigation"
+import { z } from "zod"
+import { useForm, FormProvider } from "react-hook-form"
+import { zodResolver } from "@hookform/resolvers/zod"
+import fuzzysort from "fuzzysort"
+import { toast } from "sonner"
+import { X, Rocket, Check, ChevronsUpDown, HardDriveUpload, CircleCheck } from "lucide-react"
+
+import { globalSettingsSchema, rooCodeDefaults } from "@evals/types"
+
+import { createRun } from "@/lib/server/runs"
+import { createRunSchema as formSchema, type CreateRun as FormValues } from "@/lib/schemas"
+import { cn } from "@/lib/utils"
+import { useOpenRouterModels } from "@/hooks/use-open-router-models"
+import { useExercises } from "@/hooks/use-exercises"
+import {
+	Button,
+	FormControl,
+	FormField,
+	FormItem,
+	FormLabel,
+	FormDescription,
+	FormMessage,
+	Textarea,
+	Tabs,
+	TabsList,
+	TabsTrigger,
+	MultiSelect,
+	Command,
+	CommandEmpty,
+	CommandGroup,
+	CommandInput,
+	CommandItem,
+	CommandList,
+	Popover,
+	PopoverContent,
+	PopoverTrigger,
+	ScrollArea,
+} from "@/components/ui"
+
+import { SettingsDiff } from "./settings-diff"
+
+const recommendedModels = [
+	"anthropic/claude-3.7-sonnet",
+	"anthropic/claude-3.7-sonnet:thinking",
+	"google/gemini-2.0-flash-001",
+]
+
+export function NewRun() {
+	const router = useRouter()
+
+	const [modelSearchValue, setModelSearchValue] = useState("")
+	const [modelPopoverOpen, setModelPopoverOpen] = useState(false)
+	const modelSearchResultsRef = useRef<Map<string, number>>(new Map())
+	const modelSearchValueRef = useRef("")
+	const models = useOpenRouterModels()
+
+	const exercises = useExercises()
+
+	const form = useForm<FormValues>({
+		resolver: zodResolver(formSchema),
+		defaultValues: {
+			model: "",
+			description: "",
+			suite: "full",
+			exercises: [],
+			settings: undefined,
+		},
+	})
+
+	const {
+		setValue,
+		setError,
+		clearErrors,
+		watch,
+		formState: { isSubmitting },
+	} = form
+
+	const [model, suite, settings] = watch(["model", "suite", "settings"])
+
+	const onSubmit = useCallback(
+		async (data: FormValues) => {
+			try {
+				const { id } = await createRun(data)
+				router.push(`/runs/${id}`)
+			} catch (e) {
+				toast.error(e instanceof Error ? e.message : "An unknown error occurred.")
+			}
+		},
+		[router],
+	)
+
+	const onFilterModels = useCallback(
+		(value: string, search: string) => {
+			if (modelSearchValueRef.current !== search) {
+				modelSearchValueRef.current = search
+				modelSearchResultsRef.current.clear()
+
+				for (const {
+					obj: { id },
+					score,
+				} of fuzzysort.go(search, models.data || [], {
+					key: "name",
+				})) {
+					modelSearchResultsRef.current.set(id, score)
+				}
+			}
+
+			return modelSearchResultsRef.current.get(value) ?? 0
+		},
+		[models.data],
+	)
+
+	const onSelectModel = useCallback(
+		(model: string) => {
+			setValue("model", model)
+			setModelPopoverOpen(false)
+		},
+		[setValue],
+	)
+
+	const onImportSettings = useCallback(
+		async (event: React.ChangeEvent<HTMLInputElement>) => {
+			const file = event.target.files?.[0]
+
+			if (!file) {
+				return
+			}
+
+			clearErrors("settings")
+
+			try {
+				const result = z.object({ globalSettings: globalSettingsSchema }).parse(JSON.parse(await file.text()))
+				setValue("settings", result.globalSettings)
+				event.target.value = ""
+			} catch (_error) {
+				setError("settings", { message: "Error parsing JSON file. Please check the file format." })
+			}
+		},
+		[clearErrors, setError, setValue],
+	)
+
+	return (
+		<>
+			<FormProvider {...form}>
+				<form
+					onSubmit={form.handleSubmit(onSubmit)}
+					className="flex flex-col justify-center divide-y divide-primary *:py-5">
+					<FormField
+						control={form.control}
+						name="model"
+						render={() => (
+							<FormItem>
+								<FormLabel>OpenRouter Model</FormLabel>
+								<Popover open={modelPopoverOpen} onOpenChange={setModelPopoverOpen}>
+									<PopoverTrigger asChild>
+										<Button
+											variant="input"
+											role="combobox"
+											aria-expanded={modelPopoverOpen}
+											className="flex items-center justify-between">
+											<div>
+												{models.data?.find(({ id }) => id === model)?.name || model || "Select"}
+											</div>
+											<ChevronsUpDown className="opacity-50" />
+										</Button>
+									</PopoverTrigger>
+									<PopoverContent className="p-0 w-[var(--radix-popover-trigger-width)]">
+										<Command filter={onFilterModels}>
+											<CommandInput
+												placeholder="Search"
+												value={modelSearchValue}
+												onValueChange={setModelSearchValue}
+												className="h-9"
+											/>
+											<CommandList>
+												<CommandEmpty>No model found.</CommandEmpty>
+												<CommandGroup>
+													{models.data?.map(({ id, name }) => (
+														<CommandItem key={id} value={id} onSelect={onSelectModel}>
+															{name}
+															<Check
+																className={cn(
+																	"ml-auto text-accent group-data-[selected=true]:text-accent-foreground size-4",
+																	id === model ? "opacity-100" : "opacity-0",
+																)}
+															/>
+														</CommandItem>
+													))}
+												</CommandGroup>
+											</CommandList>
+										</Command>
+									</PopoverContent>
+								</Popover>
+								<FormMessage />
+								<FormDescription className="flex flex-wrap items-center gap-2">
+									<span>Recommended:</span>
+									{recommendedModels.map((modelId) => (
+										<Button
+											key={modelId}
+											variant="link"
+											className="break-all px-0!"
+											onClick={(e) => {
+												e.preventDefault()
+												setValue("model", modelId)
+											}}>
+											{modelId}
+										</Button>
+									))}
+								</FormDescription>
+							</FormItem>
+						)}
+					/>
+
+					<FormItem>
+						<FormLabel>Import Settings</FormLabel>
+						<Button
+							type="button"
+							variant="secondary"
+							size="icon"
+							onClick={() => document.getElementById("json-upload")?.click()}>
+							<HardDriveUpload />
+						</Button>
+						<input
+							id="json-upload"
+							type="file"
+							accept="application/json"
+							className="hidden"
+							onChange={onImportSettings}
+						/>
+						{settings ? (
+							<ScrollArea className="max-h-64 border rounded-sm">
+								<>
+									<div className="flex items-center gap-1 p-2 border-b">
+										<CircleCheck className="size-4 text-ring" />
+										<div className="text-sm">
+											Imported valid Roo Code settings. Showing differences from default settings.
+										</div>
+									</div>
+									<SettingsDiff defaultSettings={rooCodeDefaults} customSettings={settings} />
+								</>
+							</ScrollArea>
+						) : (
+							<FormDescription>
+								Fully configure how Roo Code for this run using a settings file that was exported by Roo
+								Code.
+							</FormDescription>
+						)}
+						<FormMessage />
+					</FormItem>
+
+					<FormField
+						control={form.control}
+						name="suite"
+						render={() => (
+							<FormItem>
+								<FormLabel>Exercises</FormLabel>
+								<Tabs
+									defaultValue="full"
+									onValueChange={(value) => setValue("suite", value as "full" | "partial")}>
+									<TabsList>
+										<TabsTrigger value="full">All</TabsTrigger>
+										<TabsTrigger value="partial">Some</TabsTrigger>
+									</TabsList>
+								</Tabs>
+								{suite === "partial" && (
+									<MultiSelect
+										options={exercises.data?.map((path) => ({ value: path, label: path })) || []}
+										onValueChange={(value) => setValue("exercises", value)}
+										placeholder="Select"
+										variant="inverted"
+										maxCount={4}
+									/>
+								)}
+								<FormMessage />
+							</FormItem>
+						)}
+					/>
+
+					<FormField
+						control={form.control}
+						name="description"
+						render={({ field }) => (
+							<FormItem>
+								<FormLabel>Description / Notes</FormLabel>
+								<FormControl>
+									<Textarea placeholder="Optional" {...field} />
+								</FormControl>
+								<FormMessage />
+							</FormItem>
+						)}
+					/>
+
+					<div className="flex justify-end">
+						<Button size="lg" type="submit" disabled={isSubmitting}>
+							<Rocket className="size-4" />
+							Launch
+						</Button>
+					</div>
+				</form>
+			</FormProvider>
+			<Button
+				variant="default"
+				className="absolute top-4 right-12 size-12 rounded-full"
+				onClick={() => router.push("/")}>
+				<X className="size-6" />
+			</Button>
+		</>
+	)
+}

evals/apps/web/src/app/runs/new/page.tsx

@@ -0,0 +1,9 @@
+import { NewRun } from "./new-run"
+
+export default function Page() {
+	return (
+		<div className="max-w-3xl mx-auto px-12 p-12">
+			<NewRun />
+		</div>
+	)
+}

evals/apps/web/src/app/runs/new/settings-diff.tsx

@@ -0,0 +1,57 @@
+import { Fragment, HTMLAttributes } from "react"
+
+import { RooCodeSettings } from "@evals/types"
+
+import { cn } from "@/lib/utils"
+
+type SettingsDiffProps = HTMLAttributes<HTMLDivElement> & {
+	defaultSettings: RooCodeSettings
+	customSettings: RooCodeSettings
+}
+
+export function SettingsDiff({
+	customSettings: { experiments: customExperiments, ...customSettings },
+	defaultSettings: { experiments: defaultExperiments, ...defaultSettings },
+	className,
+	...props
+}: SettingsDiffProps) {
+	const defaults = { ...defaultSettings, ...defaultExperiments }
+	const custom = { ...customSettings, ...customExperiments }
+
+	return (
+		<div className={cn("grid grid-cols-3 gap-2 text-sm p-2", className)} {...props}>
+			<div className="font-medium text-muted-foreground">Setting</div>
+			<div className="font-medium text-muted-foreground">Default</div>
+			<div className="font-medium text-muted-foreground">Custom</div>
+			{Object.entries(defaults).flatMap(([key, defaultValue]) => {
+				const customValue = custom[key as keyof typeof custom]
+				const isDefault = JSON.stringify(defaultValue) === JSON.stringify(customValue)
+
+				return isDefault ? null : (
+					<SetttingDiff
+						key={key}
+						name={key}
+						defaultValue={JSON.stringify(defaultValue, null, 2)}
+						customValue={JSON.stringify(customValue, null, 2)}
+					/>
+				)
+			})}
+		</div>
+	)
+}
+
+type SettingDiffProps = HTMLAttributes<HTMLDivElement> & {
+	name: string
+	defaultValue?: string
+	customValue?: string
+}
+
+export function SetttingDiff({ name, defaultValue, customValue, ...props }: SettingDiffProps) {
+	return (
+		<Fragment {...props}>
+			<div className="overflow-hidden font-mono">{name}</div>
+			<pre className="inline text-rose-500 line-through">{defaultValue}</pre>
+			<pre className="inline text-teal-500">{customValue}</pre>
+		</Fragment>
+	)
+}

evals/apps/web/src/components/layout/header.tsx

@@ -0,0 +1,7 @@
+import { HoppingLogo } from "./logo"
+
+export const Header = () => (
+	<div className="flex items-center justify-between border-b px-12 py-6">
+		<HoppingLogo />
+	</div>
+)

evals/apps/web/src/components/layout/logo.tsx

@@ -0,0 +1,54 @@
+"use client"
+
+import { SVGProps, useEffect, useRef } from "react"
+import { useRouter } from "next/navigation"
+import { useHover } from "react-use"
+
+import { cn } from "@/lib/utils"
+
+type LogoProps = Omit<SVGProps<SVGSVGElement>, "xmlns" | "viewBox" | "onClick">
+
+export const Logo = ({ width = 50, height = 32, fill = "#fff", className, ...props }: LogoProps) => {
+	const router = useRouter()
+
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width={width}
+			height={height}
+			viewBox="90 12 100 64"
+			onClick={() => router.push("/")}
+			className={cn("logo cursor-pointer", className)}
+			{...props}>
+			<path
+				d="M171.633,15.8336l-1.7284,6.2499c-.0915.3309-.4369.5221-.7659.4239l-28.9937-8.6507c-.1928-.0575-.4016-.0167-.5586.1092l-28.7143,23.0269c-.0838.0672-.1839.1112-.2901.1276l-17.0849,2.6329c-.3163.0488-.5419.3327-.5178.6519l.0742.9817c.0237.3136.2809.5583.5953.5664l19.8448.513.2263.0063,14.6634-7.8328c.2053-.1097.455-.0936.6445.0415l10.3884,7.4053c.1629.1161.2589.3045.2571.5045l-.0876,9.826c-.0011.1272.0373.2515.11.3559l14.6133,20.9682c.1146.1644.3024.2624.5028.2624h4.626c.4615,0,.7574-.4908.542-.8989l-10.4155-19.7312c-.1019-.193-.0934-.4255.0221-.6106l5.4305-8.6994c.0591-.0947.143-.1715.2425-.222l19.415-9.8522c.1973-.1001.4332-.0861.6172.0366l5.5481,3.6981c.1007.0671.2189.1029.3399.1029h5.0407c.4881,0,.7804-.5429.5116-.9503l-13.9967-21.2171c-.2898-.4393-.962-.3331-1.1022.1741Z"
+				fill={fill}
+				strokeWidth="0"
+			/>
+		</svg>
+	)
+}
+
+export const HoppingLogo = (props: LogoProps) => {
+	const ref = useRef<SVGSVGElement>(null)
+	const logo = <Logo ref={ref} {...props} />
+	const [hoverable, hovered] = useHover(logo)
+
+	useEffect(() => {
+		const element = ref.current
+		const isHopping = element !== null && element.classList.contains("animate-hop")
+
+		if (hovered && element && !isHopping) {
+			element.classList.add("animate-hop")
+		} else if (element && isHopping) {
+			const onAnimationEnd = () => {
+				element.classList.remove("animate-hop")
+				element.removeEventListener("animationiteration", onAnimationEnd)
+			}
+
+			element.addEventListener("animationiteration", onAnimationEnd)
+		}
+	}, [hovered])
+
+	return hoverable
+}

evals/apps/web/src/components/providers/index.ts

@@ -0,0 +1,2 @@
+export { ReactQueryProvider } from "./react-query-provider"
+export { ThemeProvider } from "./theme-provider"

evals/apps/web/src/components/providers/react-query-provider.tsx

@@ -0,0 +1,8 @@
+"use client"
+
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query"
+
+export function ReactQueryProvider({ children }: { children: React.ReactNode }) {
+	const queryClient = new QueryClient()
+	return <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
+}

evals/apps/web/src/components/providers/theme-provider.tsx

@@ -0,0 +1,13 @@
+"use client"
+
+import * as React from "react"
+import { type ThemeProviderProps } from "next-themes"
+import dynamic from "next/dynamic"
+
+const NextThemesProvider = dynamic(() => import("next-themes").then((e) => e.ThemeProvider), {
+	ssr: false,
+})
+
+export function ThemeProvider({ children, ...props }: ThemeProviderProps) {
+	return <NextThemesProvider {...props}>{children}</NextThemesProvider>
+}

evals/apps/web/src/components/ui/badge.tsx

@@ -0,0 +1,36 @@
+import * as React from "react"
+import { Slot } from "@radix-ui/react-slot"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const badgeVariants = cva(
+	"inline-flex items-center justify-center rounded-sm border px-2 py-0.5 text-xs font-medium w-fit whitespace-nowrap shrink-0 [&>svg]:size-3 gap-1 [&>svg]:pointer-events-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive transition-[color,box-shadow] overflow-hidden",
+	{
+		variants: {
+			variant: {
+				default: "border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90",
+				secondary: "border-transparent bg-secondary text-secondary-foreground [a&]:hover:bg-secondary/90",
+				destructive:
+					"border-transparent bg-destructive text-white [a&]:hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/70",
+				outline: "text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground",
+			},
+		},
+		defaultVariants: {
+			variant: "default",
+		},
+	},
+)
+
+function Badge({
+	className,
+	variant,
+	asChild = false,
+	...props
+}: React.ComponentProps<"span"> & VariantProps<typeof badgeVariants> & { asChild?: boolean }) {
+	const Comp = asChild ? Slot : "span"
+
+	return <Comp data-slot="badge" className={cn(badgeVariants({ variant }), className)} {...props} />
+}
+
+export { Badge, badgeVariants }

evals/apps/web/src/components/ui/button.tsx

@@ -0,0 +1,51 @@
+import * as React from "react"
+import { Slot } from "@radix-ui/react-slot"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const buttonVariants = cva(
+	"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive hover:opacity-80 active:scale-95 cursor-pointer",
+	{
+		variants: {
+			variant: {
+				default: "bg-primary text-primary-foreground shadow-xs [&_svg]:text-accent",
+				destructive:
+					"bg-destructive text-white shadow-xs focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
+				outline:
+					"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input",
+				secondary: "bg-secondary text-secondary-foreground shadow-xs [&_svg]:text-ring",
+				ghost: "hover:bg-primary hover:text-primary-foreground",
+				link: "text-accent underline-offset-4 hover:underline h-4! px-1! rounded-none",
+				input: "bg-input text-input-foreground active:scale-100 shadow-xs",
+			},
+			size: {
+				default: "h-9 px-4 py-2 has-[>svg]:px-3",
+				sm: "h-8 gap-1.5 px-3 has-[>svg]:px-2.5 text-sm",
+				lg: "h-10 px-6 has-[>svg]:px-4 text-lg",
+				icon: "size-9",
+			},
+		},
+		defaultVariants: {
+			variant: "default",
+			size: "default",
+		},
+	},
+)
+
+function Button({
+	className,
+	variant,
+	size,
+	asChild = false,
+	...props
+}: React.ComponentProps<"button"> &
+	VariantProps<typeof buttonVariants> & {
+		asChild?: boolean
+	}) {
+	const Comp = asChild ? Slot : "button"
+
+	return <Comp data-slot="button" className={cn(buttonVariants({ variant, size, className }))} {...props} />
+}
+
+export { Button, buttonVariants }

evals/apps/web/src/components/ui/command.tsx

@@ -0,0 +1,134 @@
+"use client"
+
+import * as React from "react"
+import { Command as CommandPrimitive } from "cmdk"
+import { SearchIcon } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"
+
+function Command({ className, ...props }: React.ComponentProps<typeof CommandPrimitive>) {
+	return (
+		<CommandPrimitive
+			data-slot="command"
+			className={cn(
+				"bg-popover text-popover-foreground flex h-full w-full flex-col overflow-hidden rounded-sm",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function CommandDialog({
+	title = "Command Palette",
+	description = "Search for a command to run...",
+	children,
+	...props
+}: React.ComponentProps<typeof Dialog> & {
+	title?: string
+	description?: string
+}) {
+	return (
+		<Dialog {...props}>
+			<DialogHeader className="sr-only">
+				<DialogTitle>{title}</DialogTitle>
+				<DialogDescription>{description}</DialogDescription>
+			</DialogHeader>
+			<DialogContent className="overflow-hidden p-0">
+				<Command className="[&_[cmdk-group-heading]]:text-muted-foreground **:data-[slot=command-input-wrapper]:h-12 [&_[cmdk-group-heading]]:px-2 [&_[cmdk-group-heading]]:font-medium [&_[cmdk-group]]:px-2 [&_[cmdk-group]:not([hidden])_~[cmdk-group]]:pt-0 [&_[cmdk-input-wrapper]_svg]:h-5 [&_[cmdk-input-wrapper]_svg]:w-5 [&_[cmdk-input]]:h-12 [&_[cmdk-item]]:px-2 [&_[cmdk-item]]:py-3 [&_[cmdk-item]_svg]:h-5 [&_[cmdk-item]_svg]:w-5">
+					{children}
+				</Command>
+			</DialogContent>
+		</Dialog>
+	)
+}
+
+function CommandInput({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Input>) {
+	return (
+		<div data-slot="command-input-wrapper" className="flex h-9 items-center gap-2 border-b px-3">
+			<SearchIcon className="size-4 shrink-0 opacity-50" />
+			<CommandPrimitive.Input
+				data-slot="command-input"
+				className={cn(
+					"placeholder:text-muted-foreground flex h-10 w-full rounded-sm bg-transparent py-3 outline-hidden disabled:cursor-not-allowed disabled:opacity-50",
+					className,
+				)}
+				{...props}
+			/>
+		</div>
+	)
+}
+
+function CommandList({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.List>) {
+	return (
+		<CommandPrimitive.List
+			data-slot="command-list"
+			className={cn("max-h-[300px] scroll-py-1 overflow-x-hidden overflow-y-auto", className)}
+			{...props}
+		/>
+	)
+}
+
+function CommandEmpty({ ...props }: React.ComponentProps<typeof CommandPrimitive.Empty>) {
+	return <CommandPrimitive.Empty data-slot="command-empty" className="py-6 text-center" {...props} />
+}
+
+function CommandGroup({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Group>) {
+	return (
+		<CommandPrimitive.Group
+			data-slot="command-group"
+			className={cn(
+				"text-foreground [&_[cmdk-group-heading]]:text-muted-foreground overflow-hidden p-1 [&_[cmdk-group-heading]]:px-2 [&_[cmdk-group-heading]]:py-1.5 [&_[cmdk-group-heading]]:text-xs [&_[cmdk-group-heading]]:font-medium",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function CommandSeparator({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Separator>) {
+	return (
+		<CommandPrimitive.Separator
+			data-slot="command-separator"
+			className={cn("bg-accent/5 -mx-1 h-px", className)}
+			{...props}
+		/>
+	)
+}
+
+function CommandItem({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Item>) {
+	return (
+		<CommandPrimitive.Item
+			data-slot="command-item"
+			className={cn(
+				"data-[selected=true]:bg-accent data-[selected=true]:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex cursor-default items-center gap-2 rounded-xs px-2 py-1.5 outline-hidden select-none data-[disabled=true]:pointer-events-none data-[disabled=true]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
+				"text-foreground active:opacity-80 cursor-pointer group",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function CommandShortcut({ className, ...props }: React.ComponentProps<"span">) {
+	return (
+		<span
+			data-slot="command-shortcut"
+			className={cn("text-muted-foreground ml-auto text-xs tracking-widest", className)}
+			{...props}
+		/>
+	)
+}
+
+export {
+	Command,
+	CommandDialog,
+	CommandInput,
+	CommandList,
+	CommandEmpty,
+	CommandGroup,
+	CommandItem,
+	CommandShortcut,
+	CommandSeparator,
+}

evals/apps/web/src/components/ui/dialog.tsx

@@ -0,0 +1,110 @@
+"use client"
+
+import * as React from "react"
+import * as DialogPrimitive from "@radix-ui/react-dialog"
+import { XIcon } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+function Dialog({ ...props }: React.ComponentProps<typeof DialogPrimitive.Root>) {
+	return <DialogPrimitive.Root data-slot="dialog" {...props} />
+}
+
+function DialogTrigger({ ...props }: React.ComponentProps<typeof DialogPrimitive.Trigger>) {
+	return <DialogPrimitive.Trigger data-slot="dialog-trigger" {...props} />
+}
+
+function DialogPortal({ ...props }: React.ComponentProps<typeof DialogPrimitive.Portal>) {
+	return <DialogPrimitive.Portal data-slot="dialog-portal" {...props} />
+}
+
+function DialogClose({ ...props }: React.ComponentProps<typeof DialogPrimitive.Close>) {
+	return <DialogPrimitive.Close data-slot="dialog-close" {...props} />
+}
+
+function DialogOverlay({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Overlay>) {
+	return (
+		<DialogPrimitive.Overlay
+			data-slot="dialog-overlay"
+			className={cn(
+				"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function DialogContent({ className, children, ...props }: React.ComponentProps<typeof DialogPrimitive.Content>) {
+	return (
+		<DialogPortal data-slot="dialog-portal">
+			<DialogOverlay />
+			<DialogPrimitive.Content
+				data-slot="dialog-content"
+				className={cn(
+					"bg-background data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 fixed top-[50%] left-[50%] z-50 grid w-full max-w-[calc(100%-2rem)] translate-x-[-50%] translate-y-[-50%] gap-4 rounded-lg border p-6 shadow-lg duration-200 sm:max-w-lg",
+					className,
+				)}
+				{...props}>
+				{children}
+				<DialogPrimitive.Close className="ring-offset-background focus:ring-ring data-[state=open]:bg-accent data-[state=open]:text-muted-foreground absolute top-4 right-4 rounded-xs opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4">
+					<XIcon />
+					<span className="sr-only">Close</span>
+				</DialogPrimitive.Close>
+			</DialogPrimitive.Content>
+		</DialogPortal>
+	)
+}
+
+function DialogHeader({ className, ...props }: React.ComponentProps<"div">) {
+	return (
+		<div
+			data-slot="dialog-header"
+			className={cn("flex flex-col gap-2 text-center sm:text-left", className)}
+			{...props}
+		/>
+	)
+}
+
+function DialogFooter({ className, ...props }: React.ComponentProps<"div">) {
+	return (
+		<div
+			data-slot="dialog-footer"
+			className={cn("flex flex-col-reverse gap-2 sm:flex-row sm:justify-end", className)}
+			{...props}
+		/>
+	)
+}
+
+function DialogTitle({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Title>) {
+	return (
+		<DialogPrimitive.Title
+			data-slot="dialog-title"
+			className={cn("text-lg leading-none font-semibold", className)}
+			{...props}
+		/>
+	)
+}
+
+function DialogDescription({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Description>) {
+	return (
+		<DialogPrimitive.Description
+			data-slot="dialog-description"
+			className={cn("text-muted-foreground text-sm", className)}
+			{...props}
+		/>
+	)
+}
+
+export {
+	Dialog,
+	DialogClose,
+	DialogContent,
+	DialogDescription,
+	DialogFooter,
+	DialogHeader,
+	DialogOverlay,
+	DialogPortal,
+	DialogTitle,
+	DialogTrigger,
+}

evals/apps/web/src/components/ui/drawer.tsx

@@ -0,0 +1,98 @@
+"use client"
+
+import * as React from "react"
+import { Drawer as DrawerPrimitive } from "vaul"
+
+import { cn } from "@/lib/utils"
+
+function Drawer({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Root>) {
+	return <DrawerPrimitive.Root data-slot="drawer" {...props} />
+}
+
+function DrawerTrigger({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Trigger>) {
+	return <DrawerPrimitive.Trigger data-slot="drawer-trigger" {...props} />
+}
+
+function DrawerPortal({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Portal>) {
+	return <DrawerPrimitive.Portal data-slot="drawer-portal" {...props} />
+}
+
+function DrawerClose({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Close>) {
+	return <DrawerPrimitive.Close data-slot="drawer-close" {...props} />
+}
+
+function DrawerOverlay({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Overlay>) {
+	return (
+		<DrawerPrimitive.Overlay
+			data-slot="drawer-overlay"
+			className={cn(
+				"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function DrawerContent({ className, children, ...props }: React.ComponentProps<typeof DrawerPrimitive.Content>) {
+	return (
+		<DrawerPortal data-slot="drawer-portal">
+			<DrawerOverlay />
+			<DrawerPrimitive.Content
+				data-slot="drawer-content"
+				className={cn(
+					"group/drawer-content bg-background fixed z-50 flex h-auto flex-col",
+					"data-[vaul-drawer-direction=top]:inset-x-0 data-[vaul-drawer-direction=top]:top-0 data-[vaul-drawer-direction=top]:mb-24 data-[vaul-drawer-direction=top]:max-h-[80vh] data-[vaul-drawer-direction=top]:rounded-b-sm data-[vaul-drawer-direction=top]:border-b",
+					"data-[vaul-drawer-direction=bottom]:inset-x-0 data-[vaul-drawer-direction=bottom]:bottom-0 data-[vaul-drawer-direction=bottom]:mt-24 data-[vaul-drawer-direction=bottom]:max-h-[80vh] data-[vaul-drawer-direction=bottom]:rounded-t-sm data-[vaul-drawer-direction=bottom]:border-t",
+					"data-[vaul-drawer-direction=right]:inset-y-0 data-[vaul-drawer-direction=right]:right-0 data-[vaul-drawer-direction=right]:w-3/4 data-[vaul-drawer-direction=right]:border-l data-[vaul-drawer-direction=right]:sm:max-w-sm",
+					"data-[vaul-drawer-direction=left]:inset-y-0 data-[vaul-drawer-direction=left]:left-0 data-[vaul-drawer-direction=left]:w-3/4 data-[vaul-drawer-direction=left]:border-r data-[vaul-drawer-direction=left]:sm:max-w-sm",
+					className,
+				)}
+				{...props}>
+				<div className="bg-muted mx-auto mt-4 hidden h-2 w-[100px] shrink-0 rounded-full group-data-[vaul-drawer-direction=bottom]/drawer-content:block" />
+				{children}
+			</DrawerPrimitive.Content>
+		</DrawerPortal>
+	)
+}
+
+function DrawerHeader({ className, ...props }: React.ComponentProps<"div">) {
+	return <div data-slot="drawer-header" className={cn("flex flex-col gap-1.5 py-4", className)} {...props} />
+}
+
+function DrawerFooter({ className, ...props }: React.ComponentProps<"div">) {
+	return <div data-slot="drawer-footer" className={cn("mt-auto flex flex-col gap-2 py-4", className)} {...props} />
+}
+
+function DrawerTitle({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Title>) {
+	return (
+		<DrawerPrimitive.Title
+			data-slot="drawer-title"
+			className={cn("text-foreground font-semibold", className)}
+			{...props}
+		/>
+	)
+}
+
+function DrawerDescription({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Description>) {
+	return (
+		<DrawerPrimitive.Description
+			data-slot="drawer-description"
+			className={cn("text-muted-foreground text-sm", className)}
+			{...props}
+		/>
+	)
+}
+
+export {
+	Drawer,
+	DrawerPortal,
+	DrawerOverlay,
+	DrawerTrigger,
+	DrawerClose,
+	DrawerContent,
+	DrawerHeader,
+	DrawerFooter,
+	DrawerTitle,
+	DrawerDescription,
+}

evals/apps/web/src/components/ui/form.tsx

@@ -0,0 +1,138 @@
+"use client"
+
+import * as React from "react"
+import * as LabelPrimitive from "@radix-ui/react-label"
+import { Slot } from "@radix-ui/react-slot"
+import {
+	Controller,
+	FormProvider,
+	useFormContext,
+	useFormState,
+	type ControllerProps,
+	type FieldPath,
+	type FieldValues,
+} from "react-hook-form"
+
+import { cn } from "@/lib/utils"
+import { Label } from "@/components/ui/label"
+
+const Form = FormProvider
+
+type FormFieldContextValue<
+	TFieldValues extends FieldValues = FieldValues,
+	TName extends FieldPath<TFieldValues> = FieldPath<TFieldValues>,
+> = {
+	name: TName
+}
+
+const FormFieldContext = React.createContext<FormFieldContextValue>({} as FormFieldContextValue)
+
+const FormField = <
+	TFieldValues extends FieldValues = FieldValues,
+	TName extends FieldPath<TFieldValues> = FieldPath<TFieldValues>,
+>({
+	...props
+}: ControllerProps<TFieldValues, TName>) => {
+	return (
+		<FormFieldContext.Provider value={{ name: props.name }}>
+			<Controller {...props} />
+		</FormFieldContext.Provider>
+	)
+}
+
+const useFormField = () => {
+	const fieldContext = React.useContext(FormFieldContext)
+	const itemContext = React.useContext(FormItemContext)
+	const { getFieldState } = useFormContext()
+	const formState = useFormState({ name: fieldContext.name })
+	const fieldState = getFieldState(fieldContext.name, formState)
+
+	if (!fieldContext) {
+		throw new Error("useFormField should be used within <FormField>")
+	}
+
+	const { id } = itemContext
+
+	return {
+		id,
+		name: fieldContext.name,
+		formItemId: `${id}-form-item`,
+		formDescriptionId: `${id}-form-item-description`,
+		formMessageId: `${id}-form-item-message`,
+		...fieldState,
+	}
+}
+
+type FormItemContextValue = {
+	id: string
+}
+
+const FormItemContext = React.createContext<FormItemContextValue>({} as FormItemContextValue)
+
+function FormItem({ className, ...props }: React.ComponentProps<"div">) {
+	const id = React.useId()
+
+	return (
+		<FormItemContext.Provider value={{ id }}>
+			<div data-slot="form-item" className={cn("grid gap-2", className)} {...props} />
+		</FormItemContext.Provider>
+	)
+}
+
+function FormLabel({ className, ...props }: React.ComponentProps<typeof LabelPrimitive.Root>) {
+	const { error, formItemId } = useFormField()
+
+	return (
+		<Label
+			data-slot="form-label"
+			data-error={!!error}
+			className={cn("data-[error=true]:text-destructive", className)}
+			htmlFor={formItemId}
+			{...props}
+		/>
+	)
+}
+
+function FormControl({ ...props }: React.ComponentProps<typeof Slot>) {
+	const { error, formItemId, formDescriptionId, formMessageId } = useFormField()
+
+	return (
+		<Slot
+			data-slot="form-control"
+			id={formItemId}
+			aria-describedby={!error ? `${formDescriptionId}` : `${formDescriptionId} ${formMessageId}`}
+			aria-invalid={!!error}
+			{...props}
+		/>
+	)
+}
+
+function FormDescription({ className, ...props }: React.ComponentProps<"p">) {
+	const { formDescriptionId } = useFormField()
+
+	return (
+		<p
+			data-slot="form-description"
+			id={formDescriptionId}
+			className={cn("text-muted-foreground text-sm", className)}
+			{...props}
+		/>
+	)
+}
+
+function FormMessage({ className, ...props }: React.ComponentProps<"p">) {
+	const { error, formMessageId } = useFormField()
+	const body = error ? String(error?.message ?? "") : props.children
+
+	if (!body) {
+		return null
+	}
+
+	return (
+		<p data-slot="form-message" id={formMessageId} className={cn("text-destructive text-sm", className)} {...props}>
+			{body}
+		</p>
+	)
+}
+
+export { useFormField, Form, FormItem, FormLabel, FormControl, FormDescription, FormMessage, FormField }

evals/apps/web/src/components/ui/index.ts

@@ -0,0 +1,18 @@
+export * from "./badge"
+export * from "./button"
+export * from "./command"
+export * from "./dialog"
+export * from "./drawer"
+export * from "./form"
+export * from "./input"
+export * from "./label"
+export * from "./multi-select"
+export * from "./popover"
+export * from "./scroll-area"
+export * from "./select"
+export * from "./separator"
+export * from "./sonner"
+export * from "./table"
+export * from "./tabs"
+export * from "./textarea"
+export * from "./tooltip"

evals/apps/web/src/components/ui/input.tsx

@@ -0,0 +1,22 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Input({ className, type, ...props }: React.ComponentProps<"input">) {
+	return (
+		<input
+			type={type}
+			data-slot="input"
+			className={cn(
+				"file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground flex h-9 w-full min-w-0 rounded-sm px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50",
+				"focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px]",
+				"aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
+				"border border-input bg-input",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Input }

evals/apps/web/src/components/ui/label.tsx

@@ -0,0 +1,21 @@
+"use client"
+
+import * as React from "react"
+import * as LabelPrimitive from "@radix-ui/react-label"
+
+import { cn } from "@/lib/utils"
+
+function Label({ className, ...props }: React.ComponentProps<typeof LabelPrimitive.Root>) {
+	return (
+		<LabelPrimitive.Root
+			data-slot="label"
+			className={cn(
+				"flex items-center gap-2 leading-none font-medium select-none group-data-[disabled=true]:pointer-events-none group-data-[disabled=true]:opacity-50 peer-disabled:cursor-not-allowed peer-disabled:opacity-50",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Label }

evals/apps/web/src/components/ui/multi-select.tsx

@@ -0,0 +1,272 @@
+import * as React from "react"
+import { cva, type VariantProps } from "class-variance-authority"
+import fuzzysort from "fuzzysort"
+import { Check, X, ChevronsUpDown } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+import { Badge } from "./badge"
+import { Popover, PopoverContent, PopoverTrigger } from "./popover"
+import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "./command"
+
+/**
+ * Variants for the multi-select component to handle different styles.
+ * Uses class-variance-authority (cva) to define different styles based on "variant" prop.
+ */
+const multiSelectVariants = cva("px-2 py-1", {
+	variants: {
+		variant: {
+			default: "border-foreground/10 text-foreground bg-card hover:bg-card/80",
+			secondary: "border-foreground/10 bg-secondary text-secondary-foreground hover:bg-secondary/80",
+			destructive: "border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80",
+			inverted: "bg-background",
+		},
+	},
+	defaultVariants: {
+		variant: "default",
+	},
+})
+
+/**
+ * Props for MultiSelect component
+ */
+interface MultiSelectProps extends React.HTMLAttributes<HTMLDivElement>, VariantProps<typeof multiSelectVariants> {
+	/**
+	 * An array of option objects to be displayed in the multi-select component.
+	 * Each option object has a label and value.
+	 */
+	options: {
+		/** The text to display for the option. */
+		label: string
+		/** The unique value associated with the option. */
+		value: string
+	}[]
+
+	/**
+	 * Callback function triggered when the selected values change.
+	 * Receives an array of the new selected values.
+	 */
+	onValueChange: (value: string[]) => void
+
+	/** The default selected values when the component mounts. */
+	defaultValue?: string[]
+
+	/**
+	 * Placeholder text to be displayed when no values are selected.
+	 * Optional, defaults to "Select options".
+	 */
+	placeholder?: string
+
+	/**
+	 * Maximum number of items to display. Extra selected items will be summarized.
+	 * Optional, defaults to 3.
+	 */
+	maxCount?: number
+
+	/**
+	 * The modality of the popover. When set to true, interaction with outside elements
+	 * will be disabled and only popover content will be visible to screen readers.
+	 * Optional, defaults to false.
+	 */
+	modalPopover?: boolean
+
+	/**
+	 * If true, renders the multi-select component as a child of another component.
+	 * Optional, defaults to false.
+	 */
+	asChild?: boolean
+
+	/**
+	 * Additional class names to apply custom styles to the multi-select component.
+	 * Optional, can be used to add custom styles.
+	 */
+	className?: string
+}
+
+export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
+	(
+		{
+			options,
+			onValueChange,
+			variant,
+			defaultValue = [],
+			placeholder = "Select options",
+			maxCount = 3,
+			modalPopover = false,
+			className,
+			...props
+		},
+		ref,
+	) => {
+		const [selectedValues, setSelectedValues] = React.useState<string[]>(defaultValue)
+		const [isPopoverOpen, setIsPopoverOpen] = React.useState(false)
+
+		const handleInputKeyDown = (event: React.KeyboardEvent<HTMLInputElement>) => {
+			if (event.key === "Enter") {
+				setIsPopoverOpen(true)
+			} else if (event.key === "Backspace" && !event.currentTarget.value) {
+				const newSelectedValues = [...selectedValues]
+				newSelectedValues.pop()
+				setSelectedValues(newSelectedValues)
+				onValueChange(newSelectedValues)
+			}
+		}
+
+		const toggleOption = (option: string) => {
+			const newSelectedValues = selectedValues.includes(option)
+				? selectedValues.filter((value) => value !== option)
+				: [...selectedValues, option]
+			setSelectedValues(newSelectedValues)
+			onValueChange(newSelectedValues)
+		}
+
+		const handleTogglePopover = () => {
+			setIsPopoverOpen((prev) => !prev)
+		}
+
+		const clearExtraOptions = () => {
+			const newSelectedValues = selectedValues.slice(0, maxCount)
+			setSelectedValues(newSelectedValues)
+			onValueChange(newSelectedValues)
+		}
+
+		const searchResultsRef = React.useRef<Map<string, number>>(new Map())
+		const searchValueRef = React.useRef("")
+
+		const onSelectAll = () => {
+			const values = Array.from(searchResultsRef.current.keys())
+
+			if (
+				selectedValues.length === values.length &&
+				selectedValues.sort().join(",") === values.sort().join(",")
+			) {
+				setSelectedValues([])
+				onValueChange([])
+				return
+			}
+
+			setSelectedValues(values)
+			onValueChange(values)
+		}
+
+		const onFilter = React.useCallback(
+			(value: string, search: string) => {
+				if (searchValueRef.current !== search) {
+					searchValueRef.current = search
+					searchResultsRef.current.clear()
+
+					for (const {
+						obj: { value },
+						score,
+					} of fuzzysort.go(search, options, {
+						key: "label",
+					})) {
+						searchResultsRef.current.set(value, score)
+					}
+				}
+
+				if (value === "all") {
+					return searchResultsRef.current.size > 1 ? 0.01 : 0
+				}
+
+				return searchResultsRef.current.get(value) ?? 0
+			},
+			[options],
+		)
+
+		return (
+			<Popover open={isPopoverOpen} onOpenChange={setIsPopoverOpen} modal={modalPopover}>
+				<PopoverTrigger asChild>
+					<div
+						ref={ref}
+						{...props}
+						onClick={handleTogglePopover}
+						className={cn(
+							"flex w-full rounded-sm min-h-9 h-auto items-center justify-between [&_svg]:pointer-events-auto",
+							"font-medium border border-input bg-input hover:opacity-80 cursor-pointer",
+							className,
+						)}>
+						{selectedValues.length > 0 ? (
+							<div className="flex justify-between items-center w-full">
+								<div className="flex flex-wrap items-center gap-1 p-1">
+									{selectedValues.slice(0, maxCount).map((value) => (
+										<Badge key={value} className={cn(multiSelectVariants({ variant }))}>
+											<div className="flex items-center gap-1.5">
+												<div>{options.find((o) => o.value === value)?.label}</div>
+												<div
+													onClick={(event) => {
+														event.stopPropagation()
+														toggleOption(value)
+													}}
+													className="cursor-pointer">
+													<X className="size-4 rounded-full p-0.5 bg-accent/5" />
+												</div>
+											</div>
+										</Badge>
+									))}
+									{selectedValues.length > maxCount && (
+										<Badge className={cn("text-ring", multiSelectVariants({ variant }))}>
+											<div className="flex items-center gap-1.5">
+												<div>{`+ ${selectedValues.length - maxCount} more`}</div>
+												<div
+													onClick={(event) => {
+														event.stopPropagation()
+														clearExtraOptions()
+													}}
+													className="cursor-pointer">
+													<X className="size-4 rounded-full p-0.5 bg-ring/5" />
+												</div>
+											</div>
+										</Badge>
+									)}
+								</div>
+							</div>
+						) : (
+							<div className="flex items-center justify-between w-full mx-auto">
+								<span className="text-muted-foreground mx-3">{placeholder}</span>
+								<ChevronsUpDown className="opacity-50 size-4 mx-2" />
+							</div>
+						)}
+					</div>
+				</PopoverTrigger>
+				<PopoverContent
+					className="p-0 w-[var(--radix-popover-trigger-width)]"
+					align="start"
+					onEscapeKeyDown={() => setIsPopoverOpen(false)}>
+					<Command filter={onFilter}>
+						<CommandInput placeholder="Search" onKeyDown={handleInputKeyDown} />
+						<CommandList>
+							<CommandEmpty>No results found.</CommandEmpty>
+							<CommandGroup>
+								{options.map((option) => (
+									<CommandItem
+										key={option.value}
+										value={option.value}
+										onSelect={() => toggleOption(option.value)}
+										className="flex items-center justify-between">
+										<span>{option.label}</span>
+										<Check
+											className={cn(
+												"text-accent group-data-[selected=true]:text-accent-foreground size-4",
+												{ "opacity-0": !selectedValues.includes(option.value) },
+											)}
+										/>
+									</CommandItem>
+								))}
+								<CommandItem
+									key="all"
+									value="all"
+									onSelect={onSelectAll}
+									className="flex items-center justify-between">
+									<span>Select All</span>
+								</CommandItem>
+							</CommandGroup>
+						</CommandList>
+					</Command>
+				</PopoverContent>
+			</Popover>
+		)
+	},
+)
+
+MultiSelect.displayName = "MultiSelect"

evals/apps/web/src/components/ui/popover.tsx

@@ -0,0 +1,42 @@
+"use client"
+
+import * as React from "react"
+import * as PopoverPrimitive from "@radix-ui/react-popover"
+
+import { cn } from "@/lib/utils"
+
+function Popover({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Root>) {
+	return <PopoverPrimitive.Root data-slot="popover" {...props} />
+}
+
+function PopoverTrigger({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Trigger>) {
+	return <PopoverPrimitive.Trigger data-slot="popover-trigger" {...props} />
+}
+
+function PopoverContent({
+	className,
+	align = "center",
+	sideOffset = 4,
+	...props
+}: React.ComponentProps<typeof PopoverPrimitive.Content>) {
+	return (
+		<PopoverPrimitive.Portal>
+			<PopoverPrimitive.Content
+				data-slot="popover-content"
+				align={align}
+				sideOffset={sideOffset}
+				className={cn(
+					"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-72 origin-(--radix-popover-content-transform-origin) rounded-sm border p-4 shadow-md outline-hidden",
+					className,
+				)}
+				{...props}
+			/>
+		</PopoverPrimitive.Portal>
+	)
+}
+
+function PopoverAnchor({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Anchor>) {
+	return <PopoverPrimitive.Anchor data-slot="popover-anchor" {...props} />
+}
+
+export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor }

evals/apps/web/src/components/ui/scroll-area.tsx

@@ -0,0 +1,51 @@
+"use client"
+
+import * as React from "react"
+import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"
+
+import { cn } from "@/lib/utils"
+
+type ScrollAreaProps = React.ComponentProps<typeof ScrollAreaPrimitive.Root> & {
+	viewportRef?: React.RefObject<HTMLDivElement | null>
+}
+
+function ScrollArea({ className, children, viewportRef, ...props }: ScrollAreaProps) {
+	return (
+		<ScrollAreaPrimitive.Root data-slot="scroll-area" className={cn("relative", className)} {...props}>
+			<ScrollAreaPrimitive.Viewport
+				ref={viewportRef}
+				data-slot="scroll-area-viewport"
+				className="ring-ring/10 dark:ring-ring/20 dark:outline-ring/40 outline-ring/50 size-full rounded-[inherit] transition-[color,box-shadow] focus-visible:ring-4 focus-visible:outline-1">
+				{children}
+			</ScrollAreaPrimitive.Viewport>
+			<ScrollBar />
+			<ScrollAreaPrimitive.Corner />
+		</ScrollAreaPrimitive.Root>
+	)
+}
+
+function ScrollBar({
+	className,
+	orientation = "vertical",
+	...props
+}: React.ComponentProps<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>) {
+	return (
+		<ScrollAreaPrimitive.ScrollAreaScrollbar
+			data-slot="scroll-area-scrollbar"
+			orientation={orientation}
+			className={cn(
+				"flex touch-none p-px transition-colors select-none",
+				orientation === "vertical" && "h-full w-2.5 border-l border-l-transparent",
+				orientation === "horizontal" && "h-2.5 flex-col border-t border-t-transparent",
+				className,
+			)}
+			{...props}>
+			<ScrollAreaPrimitive.ScrollAreaThumb
+				data-slot="scroll-area-thumb"
+				className="bg-border relative flex-1 rounded-full"
+			/>
+		</ScrollAreaPrimitive.ScrollAreaScrollbar>
+	)
+}
+
+export { ScrollArea, ScrollBar }

evals/apps/web/src/components/ui/select.tsx

@@ -0,0 +1,156 @@
+"use client"
+
+import * as React from "react"
+import * as SelectPrimitive from "@radix-ui/react-select"
+import { Check, ChevronDown, ChevronUp } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+function Select({ ...props }: React.ComponentProps<typeof SelectPrimitive.Root>) {
+	return <SelectPrimitive.Root data-slot="select" {...props} />
+}
+
+function SelectGroup({ ...props }: React.ComponentProps<typeof SelectPrimitive.Group>) {
+	return <SelectPrimitive.Group data-slot="select-group" {...props} />
+}
+
+function SelectValue({ ...props }: React.ComponentProps<typeof SelectPrimitive.Value>) {
+	return <SelectPrimitive.Value data-slot="select-value" {...props} />
+}
+
+function SelectTrigger({
+	className,
+	size = "default",
+	children,
+	...props
+}: React.ComponentProps<typeof SelectPrimitive.Trigger> & {
+	size?: "sm" | "default"
+}) {
+	return (
+		<SelectPrimitive.Trigger
+			data-slot="select-trigger"
+			data-size={size}
+			className={cn(
+				"data-[placeholder]:text-muted-foreground [&_svg:not([class*='text-'])]:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex w-fit items-center justify-between gap-2 rounded-sm px-3 py-2 whitespace-nowrap shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 data-[size=default]:h-9 data-[size=sm]:h-8 *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-2 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
+				"border border-input bg-input hover:opacity-80 cursor-pointer",
+				className,
+			)}
+			{...props}>
+			{children}
+			<SelectPrimitive.Icon asChild>
+				<ChevronDown className="size-4 opacity-50" />
+			</SelectPrimitive.Icon>
+		</SelectPrimitive.Trigger>
+	)
+}
+
+function SelectContent({
+	className,
+	children,
+	position = "popper",
+	...props
+}: React.ComponentProps<typeof SelectPrimitive.Content>) {
+	return (
+		<SelectPrimitive.Portal>
+			<SelectPrimitive.Content
+				data-slot="select-content"
+				className={cn(
+					"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 relative z-50 max-h-(--radix-select-content-available-height) min-w-[8rem] origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-sm shadow-md",
+					position === "popper" &&
+						"data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
+					className,
+				)}
+				position={position}
+				{...props}>
+				<SelectScrollUpButton />
+				<SelectPrimitive.Viewport
+					className={cn(
+						"p-1",
+						position === "popper" &&
+							"h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)] scroll-my-1",
+					)}>
+					{children}
+				</SelectPrimitive.Viewport>
+				<SelectScrollDownButton />
+			</SelectPrimitive.Content>
+		</SelectPrimitive.Portal>
+	)
+}
+
+function SelectLabel({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.Label>) {
+	return (
+		<SelectPrimitive.Label
+			data-slot="select-label"
+			className={cn("text-muted-foreground px-2 py-1.5 text-xs", className)}
+			{...props}
+		/>
+	)
+}
+
+function SelectItem({ className, children, ...props }: React.ComponentProps<typeof SelectPrimitive.Item>) {
+	return (
+		<SelectPrimitive.Item
+			data-slot="select-item"
+			className={cn(
+				"focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex w-full cursor-default items-center gap-2 rounded-xs py-1.5 pr-8 pl-2 outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
+				"text-foreground active:opacity-80 cursor-pointer group",
+				className,
+			)}
+			{...props}>
+			<span className="absolute right-2 flex size-3.5 items-center justify-center">
+				<SelectPrimitive.ItemIndicator>
+					<Check className="text-accent group-focus:text-accent-foreground size-4" />
+				</SelectPrimitive.ItemIndicator>
+			</span>
+			<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+		</SelectPrimitive.Item>
+	)
+}
+
+function SelectSeparator({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.Separator>) {
+	return (
+		<SelectPrimitive.Separator
+			data-slot="select-separator"
+			className={cn("bg-border pointer-events-none -mx-1 my-1 h-px", className)}
+			{...props}
+		/>
+	)
+}
+
+function SelectScrollUpButton({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.ScrollUpButton>) {
+	return (
+		<SelectPrimitive.ScrollUpButton
+			data-slot="select-scroll-up-button"
+			className={cn("flex cursor-default items-center justify-center py-1", className)}
+			{...props}>
+			<ChevronUp className="size-4" />
+		</SelectPrimitive.ScrollUpButton>
+	)
+}
+
+function SelectScrollDownButton({
+	className,
+	...props
+}: React.ComponentProps<typeof SelectPrimitive.ScrollDownButton>) {
+	return (
+		<SelectPrimitive.ScrollDownButton
+			data-slot="select-scroll-down-button"
+			className={cn("flex cursor-default items-center justify-center py-1", className)}
+			{...props}>
+			<ChevronDown className="size-4" />
+		</SelectPrimitive.ScrollDownButton>
+	)
+}
+
+export {
+	Select,
+	SelectContent,
+	SelectGroup,
+	SelectItem,
+	SelectLabel,
+	SelectScrollDownButton,
+	SelectScrollUpButton,
+	SelectSeparator,
+	SelectTrigger,
+	SelectValue,
+}

evals/apps/web/src/components/ui/separator.tsx

@@ -0,0 +1,28 @@
+"use client"
+
+import * as React from "react"
+import * as SeparatorPrimitive from "@radix-ui/react-separator"
+
+import { cn } from "@/lib/utils"
+
+function Separator({
+	className,
+	orientation = "horizontal",
+	decorative = true,
+	...props
+}: React.ComponentProps<typeof SeparatorPrimitive.Root>) {
+	return (
+		<SeparatorPrimitive.Root
+			data-slot="separator-root"
+			decorative={decorative}
+			orientation={orientation}
+			className={cn(
+				"bg-border shrink-0 data-[orientation=horizontal]:h-px data-[orientation=horizontal]:w-full data-[orientation=vertical]:h-full data-[orientation=vertical]:w-px",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Separator }

evals/apps/web/src/components/ui/sonner.tsx

@@ -0,0 +1,25 @@
+"use client"
+
+import { useTheme } from "next-themes"
+import { Toaster as Sonner, ToasterProps } from "sonner"
+
+const Toaster = ({ ...props }: ToasterProps) => {
+	const { theme = "system" } = useTheme()
+
+	return (
+		<Sonner
+			theme={theme as ToasterProps["theme"]}
+			className="toaster group"
+			style={
+				{
+					"--normal-bg": "var(--popover)",
+					"--normal-text": "var(--popover-foreground)",
+					"--normal-border": "var(--border)",
+				} as React.CSSProperties
+			}
+			{...props}
+		/>
+	)
+}
+
+export { Toaster }

evals/apps/web/src/components/ui/table.tsx

@@ -0,0 +1,75 @@
+"use client"
+
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Table({ className, ...props }: React.ComponentProps<"table">) {
+	return (
+		<div data-slot="table-container" className="relative w-full overflow-x-auto">
+			<table data-slot="table" className={cn("w-full caption-bottom text-sm", className)} {...props} />
+		</div>
+	)
+}
+
+function TableHeader({ className, ...props }: React.ComponentProps<"thead">) {
+	return <thead data-slot="table-header" className={cn("[&_tr]:border-b", className)} {...props} />
+}
+
+function TableBody({ className, ...props }: React.ComponentProps<"tbody">) {
+	return <tbody data-slot="table-body" className={cn("[&_tr:last-child]:border-0", className)} {...props} />
+}
+
+function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) {
+	return (
+		<tfoot
+			data-slot="table-footer"
+			className={cn("bg-muted/50 border-t font-medium [&>tr]:last:border-b-0", className)}
+			{...props}
+		/>
+	)
+}
+
+function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
+	return (
+		<tr
+			data-slot="table-row"
+			className={cn("hover:bg-accent/5 data-[state=selected]:bg-muted border-b transition-colors", className)}
+			{...props}
+		/>
+	)
+}
+
+function TableHead({ className, ...props }: React.ComponentProps<"th">) {
+	return (
+		<th
+			data-slot="table-head"
+			className={cn(
+				"text-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function TableCell({ className, ...props }: React.ComponentProps<"td">) {
+	return (
+		<td
+			data-slot="table-cell"
+			className={cn(
+				"p-2 align-middle whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function TableCaption({ className, ...props }: React.ComponentProps<"caption">) {
+	return (
+		<caption data-slot="table-caption" className={cn("text-muted-foreground mt-4 text-sm", className)} {...props} />
+	)
+}
+
+export { Table, TableHeader, TableBody, TableFooter, TableHead, TableRow, TableCell, TableCaption }

evals/apps/web/src/components/ui/tabs.tsx

@@ -0,0 +1,122 @@
+"use client"
+
+import * as React from "react"
+import { useEffect, useRef, useState } from "react"
+import * as TabsPrimitive from "@radix-ui/react-tabs"
+
+import { cn } from "@/lib/utils"
+
+const Tabs = TabsPrimitive.Root
+
+const TabsList = React.forwardRef<
+	React.ComponentRef<typeof TabsPrimitive.List>,
+	React.ComponentPropsWithoutRef<typeof TabsPrimitive.List>
+>(({ className, ...props }, ref) => {
+	const [indicatorStyle, setIndicatorStyle] = useState({
+		left: 0,
+		top: 0,
+		width: 0,
+		height: 0,
+	})
+
+	const tabsListRef = useRef<HTMLDivElement | null>(null)
+
+	const updateIndicator = React.useCallback(() => {
+		if (!tabsListRef.current) {
+			return
+		}
+
+		const activeTab = tabsListRef.current.querySelector<HTMLElement>('[data-state="active"]')
+
+		if (!activeTab) {
+			return
+		}
+
+		const activeRect = activeTab.getBoundingClientRect()
+		const tabsRect = tabsListRef.current.getBoundingClientRect()
+
+		requestAnimationFrame(() => {
+			setIndicatorStyle({
+				left: activeRect.left - tabsRect.left,
+				top: activeRect.top - tabsRect.top,
+				width: activeRect.width,
+				height: activeRect.height,
+			})
+		})
+	}, [])
+
+	useEffect(() => {
+		const timeoutId = setTimeout(updateIndicator, 0)
+
+		window.addEventListener("resize", updateIndicator)
+		const observer = new MutationObserver(updateIndicator)
+
+		if (tabsListRef.current) {
+			observer.observe(tabsListRef.current, {
+				attributes: true,
+				childList: true,
+				subtree: true,
+			})
+		}
+
+		return () => {
+			clearTimeout(timeoutId)
+			window.removeEventListener("resize", updateIndicator)
+			observer.disconnect()
+		}
+	}, [updateIndicator])
+
+	return (
+		<div className="relative" ref={tabsListRef}>
+			<TabsPrimitive.List
+				ref={ref}
+				className={cn(
+					"relative inline-flex items-center justify-center rounded-sm bg-primary p-0.5 text-muted-foreground",
+					className,
+				)}
+				{...props}
+			/>
+			<div
+				className={cn(
+					"absolute rounded-sm transition-all duration-300 ease-in-out pointer-events-none",
+					"bg-accent/5",
+				)}
+				style={indicatorStyle}
+			/>
+		</div>
+	)
+})
+TabsList.displayName = TabsPrimitive.List.displayName
+
+const TabsTrigger = React.forwardRef<
+	React.ComponentRef<typeof TabsPrimitive.Trigger>,
+	React.ComponentPropsWithoutRef<typeof TabsPrimitive.Trigger>
+>(({ className, ...props }, ref) => (
+	<TabsPrimitive.Trigger
+		ref={ref}
+		className={cn(
+			"inline-flex items-center justify-center whitespace-nowrap rounded-sm px-3 py-1 ring-offset-background transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 z-10",
+			"data-[state=active]:text-accent data-[state=active]:font-medium cursor-pointer",
+			className,
+		)}
+		{...props}
+	/>
+))
+TabsTrigger.displayName = TabsPrimitive.Trigger.displayName
+
+const TabsContent = React.forwardRef<
+	React.ComponentRef<typeof TabsPrimitive.Content>,
+	React.ComponentPropsWithoutRef<typeof TabsPrimitive.Content>
+>(({ className, ...props }, ref) => (
+	<TabsPrimitive.Content
+		ref={ref}
+		className={cn(
+			"mt-2 ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2",
+			className,
+		)}
+		{...props}
+	/>
+))
+TabsContent.displayName = TabsPrimitive.Content.displayName
+
+export { Tabs, TabsContent, TabsList, TabsTrigger }

evals/apps/web/src/components/ui/textarea.tsx

@@ -0,0 +1,19 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
+	return (
+		<textarea
+			data-slot="textarea"
+			className={cn(
+				"placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex field-sizing-content min-h-16 w-full rounded-sm px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50",
+				"border border-input bg-input",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Textarea }

evals/apps/web/src/components/ui/tooltip.tsx

@@ -0,0 +1,47 @@
+"use client"
+
+import * as React from "react"
+import * as TooltipPrimitive from "@radix-ui/react-tooltip"
+
+import { cn } from "@/lib/utils"
+
+function TooltipProvider({ delayDuration = 0, ...props }: React.ComponentProps<typeof TooltipPrimitive.Provider>) {
+	return <TooltipPrimitive.Provider data-slot="tooltip-provider" delayDuration={delayDuration} {...props} />
+}
+
+function Tooltip({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Root>) {
+	return (
+		<TooltipProvider>
+			<TooltipPrimitive.Root data-slot="tooltip" {...props} />
+		</TooltipProvider>
+	)
+}
+
+function TooltipTrigger({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Trigger>) {
+	return <TooltipPrimitive.Trigger data-slot="tooltip-trigger" {...props} />
+}
+
+function TooltipContent({
+	className,
+	sideOffset = 0,
+	children,
+	...props
+}: React.ComponentProps<typeof TooltipPrimitive.Content>) {
+	return (
+		<TooltipPrimitive.Portal>
+			<TooltipPrimitive.Content
+				data-slot="tooltip-content"
+				sideOffset={sideOffset}
+				className={cn(
+					"bg-primary text-primary-foreground animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit origin-(--radix-tooltip-content-transform-origin) rounded-sm px-3 py-1.5 text-xs text-balance",
+					className,
+				)}
+				{...props}>
+				{children}
+				<TooltipPrimitive.Arrow className="bg-primary fill-primary z-50 size-2.5 translate-y-[calc(-50%_-_2px)] rotate-45 rounded-[2px]" />
+			</TooltipPrimitive.Content>
+		</TooltipPrimitive.Portal>
+	)
+}
+
+export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider }

evals/apps/web/src/hooks/use-event-source.ts

@@ -0,0 +1,57 @@
+import { useCallback, useEffect, useRef, useState } from "react"
+
+export type EventSourceStatus = "waiting" | "connected" | "error"
+
+export type EventSourceEvent = Event & { data: string }
+
+type UseEventSourceOptions = {
+	url: string
+	withCredentials?: boolean
+	onMessage: (event: MessageEvent) => void
+}
+
+export function useEventSource({ url, withCredentials, onMessage }: UseEventSourceOptions) {
+	const sourceRef = useRef<EventSource | null>(null)
+	const statusRef = useRef<EventSourceStatus>("waiting")
+	const [status, setStatus] = useState<EventSourceStatus>("waiting")
+	const handleMessage = useCallback((event: MessageEvent) => onMessage(event), [onMessage])
+
+	const createEventSource = useCallback(() => {
+		sourceRef.current = new EventSource(url, { withCredentials })
+
+		sourceRef.current.onopen = () => {
+			statusRef.current = "connected"
+			setStatus("connected")
+		}
+
+		sourceRef.current.onmessage = (event) => {
+			handleMessage(event)
+		}
+
+		sourceRef.current.onerror = () => {
+			statusRef.current = "error"
+			setStatus("error")
+			// sourceRef.current?.close()
+			// sourceRef.current = null
+		}
+	}, [url, withCredentials, handleMessage])
+
+	useEffect(() => {
+		createEventSource()
+
+		setTimeout(() => {
+			if (statusRef.current === "waiting") {
+				sourceRef.current?.close()
+				sourceRef.current = null
+				createEventSource()
+			}
+		}, 100)
+
+		return () => {
+			sourceRef.current?.close()
+			sourceRef.current = null
+		}
+	}, [createEventSource])
+
+	return status
+}

evals/apps/web/src/hooks/use-exercises.ts

@@ -0,0 +1,5 @@
+import { useQuery } from "@tanstack/react-query"
+
+import { getExercises } from "@/lib/server/exercises"
+
+export const useExercises = () => useQuery({ queryKey: ["exercises"], queryFn: getExercises })

evals/apps/web/src/hooks/use-open-router-models.ts

@@ -0,0 +1,36 @@
+import { z } from "zod"
+import { useQuery } from "@tanstack/react-query"
+
+export const openRouterModelSchema = z.object({
+	id: z.string(),
+	name: z.string(),
+	description: z.string(),
+	created: z.number(),
+	context_length: z.number(),
+})
+
+export type OpenRouterModel = z.infer<typeof openRouterModelSchema>
+
+export const getOpenRouterModels = async () => {
+	const response = await fetch("https://openrouter.ai/api/v1/models")
+
+	if (!response.ok) {
+		console.error("Failed to fetch OpenRouter models")
+		return []
+	}
+
+	const result = z.object({ data: z.array(openRouterModelSchema) }).safeParse(await response.json())
+
+	if (!result.success) {
+		console.error(result.error)
+		return []
+	}
+
+	return result.data.data.sort((a, b) => a.name.localeCompare(b.name))
+}
+
+export const useOpenRouterModels = () =>
+	useQuery<OpenRouterModel[]>({
+		queryKey: ["getOpenRouterModels"],
+		queryFn: getOpenRouterModels,
+	})

evals/apps/web/src/hooks/use-process-tree.ts

@@ -0,0 +1,10 @@
+import { useQuery } from "@tanstack/react-query"
+
+import { getProcessList } from "@/lib/server/processes"
+
+export const useProcessList = (pid: number | null) =>
+	useQuery({
+		queryKey: ["process-tree", pid],
+		queryFn: () => (pid ? getProcessList(pid) : []),
+		enabled: !!pid,
+	})

evals/apps/web/src/hooks/use-run-status.ts

@@ -0,0 +1,80 @@
+import { useState, useCallback, useRef } from "react"
+import { useQuery, keepPreviousData } from "@tanstack/react-query"
+
+import { RooCodeEventName, taskEventSchema } from "@evals/types"
+import { Run } from "@evals/db"
+
+import { getTasks } from "@/lib/server/tasks"
+import { useEventSource } from "@/hooks/use-event-source"
+
+export const useRunStatus = (run: Run) => {
+	const [tasksUpdatedAt, setTasksUpdatedAt] = useState<number>()
+	const outputRef = useRef<Map<number, string[]>>(new Map())
+	const [outputCounts, setOutputCounts] = useState<Record<number, number>>({})
+
+	const { data: tasks } = useQuery({
+		queryKey: ["run", run.id, tasksUpdatedAt],
+		queryFn: async () => getTasks(run.id),
+		placeholderData: keepPreviousData,
+		refetchInterval: 10_000,
+	})
+
+	const url = `/api/runs/${run.id}/stream`
+
+	const onMessage = useCallback((messageEvent: MessageEvent) => {
+		let data
+
+		try {
+			data = JSON.parse(messageEvent.data)
+		} catch (_) {
+			console.log(`invalid JSON: ${messageEvent.data}`)
+			return
+		}
+
+		const result = taskEventSchema.safeParse(data)
+
+		if (!result.success) {
+			console.log(`unrecognized messageEvent.data: ${messageEvent.data}`)
+			return
+		}
+
+		const { eventName, payload, taskId } = result.data
+
+		if (!taskId) {
+			console.log(`no taskId: ${messageEvent.data}`)
+			return
+		}
+
+		switch (eventName) {
+			case RooCodeEventName.TaskStarted:
+			case RooCodeEventName.TaskCompleted:
+			case RooCodeEventName.TaskAborted:
+				setTasksUpdatedAt(Date.now())
+				break
+			case RooCodeEventName.Message: {
+				const [
+					{
+						message: { text },
+					},
+				] = payload
+
+				if (text) {
+					outputRef.current.set(taskId, [...(outputRef.current.get(taskId) || []), text])
+					const outputCounts: Record<number, number> = {}
+
+					for (const [taskId, messages] of outputRef.current.entries()) {
+						outputCounts[taskId] = messages.length
+					}
+
+					setOutputCounts(outputCounts)
+				}
+
+				break
+			}
+		}
+	}, [])
+
+	const status = useEventSource({ url, onMessage })
+
+	return { tasks, status, output: outputRef.current, outputCounts }
+}

evals/apps/web/src/lib/format-currency.ts

@@ -0,0 +1,6 @@
+const formatter = new Intl.NumberFormat("en-US", {
+	style: "currency",
+	currency: "USD",
+})
+
+export const formatCurrency = (amount: number) => formatter.format(amount)

evals/apps/web/src/lib/format-duration.ts

@@ -0,0 +1,22 @@
+export const formatDuration = (durationMs: number) => {
+	const seconds = Math.floor(durationMs / 1000)
+	const hours = Math.floor(seconds / 3600)
+	const minutes = Math.floor((seconds % 3600) / 60)
+	const remainingSeconds = seconds % 60
+
+	const parts = []
+
+	if (hours > 0) {
+		parts.push(`${hours}h`)
+	}
+
+	if (minutes > 0) {
+		parts.push(`${minutes}m`)
+	}
+
+	if (remainingSeconds > 0 || parts.length === 0) {
+		parts.push(`${remainingSeconds}s`)
+	}
+
+	return parts.join(" ")
+}

evals/apps/web/src/lib/format-tokens.ts

@@ -0,0 +1,7 @@
+export const formatTokens = (tokens: number) => {
+	if (tokens < 1000) {
+		return tokens.toString()
+	}
+
+	return `${(tokens / 1000).toFixed(1)}k`
+}

evals/apps/web/src/lib/index.ts

@@ -0,0 +1,3 @@
+export { formatCurrency } from "./format-currency"
+export { formatDuration } from "./format-duration"
+export { formatTokens } from "./format-tokens"

evals/apps/web/src/lib/schemas.ts

@@ -0,0 +1,22 @@
+import { z } from "zod"
+
+import { globalSettingsSchema } from "@evals/types"
+
+/**
+ * CreateRun
+ */
+
+export const createRunSchema = z
+	.object({
+		model: z.string().min(1, { message: "Model is required." }),
+		description: z.string().optional(),
+		suite: z.enum(["full", "partial"]),
+		exercises: z.array(z.string()).optional(),
+		settings: globalSettingsSchema.optional(),
+	})
+	.refine((data) => data.suite === "full" || (data.exercises || []).length > 0, {
+		message: "Exercises are required when running a partial suite.",
+		path: ["exercises"],
+	})
+
+export type CreateRun = z.infer<typeof createRunSchema>

evals/apps/web/src/lib/server/exercises.ts

@@ -0,0 +1,38 @@
+"use server"
+
+import * as fs from "fs/promises"
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+import { ExerciseLanguage, exerciseLanguages } from "@evals/types"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const listDirectories = async (relativePath: string) => {
+	try {
+		const targetPath = path.resolve(__dirname, relativePath)
+		const entries = await fs.readdir(targetPath, { withFileTypes: true })
+		return entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith(".")).map((entry) => entry.name)
+	} catch (error) {
+		console.error(`Error listing directories at ${relativePath}:`, error)
+		return []
+	}
+}
+
+// __dirname = <repo>/evals/apps/web/src/lib/server
+const EXERCISES_BASE_PATH = path.resolve(__dirname, "../../../../../../../evals")
+
+export const getExercises = async () => {
+	const result = await Promise.all(
+		exerciseLanguages.map(async (language) => {
+			const languagePath = path.join(EXERCISES_BASE_PATH, language)
+			const exercises = await listDirectories(languagePath)
+			return exercises.map((exercise) => `${language}/${exercise}`)
+		}),
+	)
+
+	return result.flat()
+}
+
+export const getExercisesForLanguage = async (language: ExerciseLanguage) =>
+	listDirectories(path.join(EXERCISES_BASE_PATH, language))

evals/apps/web/src/lib/server/processes.ts

@@ -0,0 +1,46 @@
+"use server"
+
+import psTree from "ps-tree"
+import { exec } from "child_process"
+
+export const getProcessList = async (pid: number) => {
+	const promise = new Promise<string>((resolve, reject) => {
+		exec(`ps -p ${pid} -o pid=`, (err, stdout, stderr) => {
+			if (err) {
+				reject(stderr)
+			}
+
+			resolve(stdout)
+		})
+	})
+
+	try {
+		await promise
+	} catch (_) {
+		return null
+	}
+
+	return new Promise<number[]>((resolve, reject) => {
+		psTree(pid, (err, children) => {
+			if (err) {
+				reject(err)
+			}
+
+			resolve(children.map((p) => parseInt(p.PID)))
+		})
+	})
+}
+
+export const killProcessTree = async (pid: number) => {
+	const descendants = await getProcessList(pid)
+
+	if (descendants === null) {
+		return
+	}
+
+	if (descendants.length > 0) {
+		await exec(`kill -9 ${descendants.join(" ")}`)
+	}
+
+	await exec(`kill -9 ${pid}`)
+}

evals/apps/web/src/lib/server/runs.ts

@@ -0,0 +1,60 @@
+"use server"
+
+import { spawn } from "child_process"
+import path from "path"
+import os from "os"
+import fs from "fs"
+
+import { revalidatePath } from "next/cache"
+import pMap from "p-map"
+
+import { ExerciseLanguage, exerciseLanguages } from "@evals/types"
+import * as db from "@evals/db"
+
+import { CreateRun } from "@/lib/schemas"
+import { getExercisesForLanguage } from "./exercises"
+
+export async function createRun({ suite, exercises = [], ...values }: CreateRun) {
+	const run = await db.createRun({
+		...values,
+		socketPath: path.join(os.tmpdir(), `roo-code-evals-${crypto.randomUUID()}.sock`),
+	})
+
+	if (suite === "partial") {
+		for (const path of exercises) {
+			const [language, exercise] = path.split("/")
+
+			if (!language || !exercise) {
+				throw new Error("Invalid exercise path: " + path)
+			}
+
+			await db.createTask({ ...values, runId: run.id, language: language as ExerciseLanguage, exercise })
+		}
+	} else {
+		for (const language of exerciseLanguages) {
+			const exercises = await getExercisesForLanguage(language)
+
+			await pMap(exercises, (exercise) => db.createTask({ ...values, runId: run.id, language, exercise }), {
+				concurrency: 10,
+			})
+		}
+	}
+
+	revalidatePath("/runs")
+
+	try {
+		const logFile = fs.openSync(`/tmp/roo-code-evals-${run.id}.log`, "a")
+
+		const process = spawn("pnpm", ["--filter", "@evals/cli", "dev", "run", "all", "--runId", run.id.toString()], {
+			detached: true,
+			stdio: ["ignore", logFile, logFile],
+		})
+
+		process.unref()
+		await db.updateRun(run.id, { pid: process.pid })
+	} catch (error) {
+		console.error(error)
+	}
+
+	return run
+}

evals/apps/web/src/lib/server/sse-stream.ts

@@ -0,0 +1,37 @@
+export class SSEStream {
+	private readonly _stream: TransformStream
+	private readonly _writer: WritableStreamDefaultWriter
+	private readonly _encoder: TextEncoder
+
+	constructor() {
+		this._stream = new TransformStream()
+		this._writer = this._stream.writable.getWriter()
+		this._encoder = new TextEncoder()
+	}
+
+	public async write(data: string | object) {
+		try {
+			const buffer = typeof data === "object" ? JSON.stringify(data) : data
+			await this._writer.write(this._encoder.encode(`data: ${buffer}\n\n`))
+			return true
+		} catch (error) {
+			console.error("[SSEStream#write]", error)
+			this.close().catch(() => {})
+			return false
+		}
+	}
+
+	public close() {
+		return this._writer.close()
+	}
+
+	public getResponse() {
+		return new Response(this._stream.readable, {
+			headers: {
+				"Content-Type": "text/event-stream",
+				Connection: "keep-alive",
+				"Cache-Control": "no-cache, no-transform",
+			},
+		})
+	}
+}

evals/apps/web/src/lib/server/tasks.ts

@@ -0,0 +1,11 @@
+"use server"
+
+import { revalidatePath } from "next/cache"
+
+import * as db from "@evals/db"
+
+export async function getTasks(runId: number) {
+	const tasks = await db.getTasks(runId)
+	revalidatePath(`/runs/${runId}`)
+	return tasks
+}

evals/apps/web/src/lib/utils.ts

@@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx"
+import { twMerge } from "tailwind-merge"
+
+export function cn(...inputs: ClassValue[]) {
+	return twMerge(clsx(inputs))
+}

evals/apps/web/tsconfig.json

@@ -0,0 +1,9 @@
+{
+	"extends": "@evals/typescript-config/nextjs.json",
+	"compilerOptions": {
+		"plugins": [{ "name": "next" }],
+		"paths": { "@/*": ["./src/*"] }
+	},
+	"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+	"exclude": ["node_modules"]
+}

evals/config/eslint/base.js

@@ -0,0 +1,32 @@
+import js from "@eslint/js"
+import eslintConfigPrettier from "eslint-config-prettier"
+import turboPlugin from "eslint-plugin-turbo"
+import tseslint from "typescript-eslint"
+import onlyWarn from "eslint-plugin-only-warn"
+
+/**
+ * A shared ESLint configuration for the repository.
+ *
+ * @type {import("eslint").Linter.Config[]}
+ * */
+export const config = [
+	js.configs.recommended,
+	eslintConfigPrettier,
+	...tseslint.configs.recommended,
+	{
+		plugins: {
+			turbo: turboPlugin,
+		},
+		rules: {
+			"turbo/no-undeclared-env-vars": "warn",
+		},
+	},
+	{
+		plugins: {
+			onlyWarn,
+		},
+	},
+	{
+		ignores: ["dist/**"],
+	},
+]

evals/config/eslint/next.js

@@ -0,0 +1,50 @@
+import js from "@eslint/js"
+import eslintConfigPrettier from "eslint-config-prettier"
+import tseslint from "typescript-eslint"
+import pluginReactHooks from "eslint-plugin-react-hooks"
+import pluginReact from "eslint-plugin-react"
+import globals from "globals"
+import pluginNext from "@next/eslint-plugin-next"
+
+import { config as baseConfig } from "./base.js"
+
+/**
+ * A custom ESLint configuration for libraries that use Next.js.
+ *
+ * @type {import("eslint").Linter.Config[]}
+ * */
+export const nextJsConfig = [
+	...baseConfig,
+	js.configs.recommended,
+	eslintConfigPrettier,
+	...tseslint.configs.recommended,
+	{
+		...pluginReact.configs.flat.recommended,
+		languageOptions: {
+			...pluginReact.configs.flat.recommended.languageOptions,
+			globals: {
+				...globals.serviceworker,
+			},
+		},
+	},
+	{
+		plugins: {
+			"@next/next": pluginNext,
+		},
+		rules: {
+			...pluginNext.configs.recommended.rules,
+			...pluginNext.configs["core-web-vitals"].rules,
+		},
+	},
+	{
+		plugins: {
+			"react-hooks": pluginReactHooks,
+		},
+		settings: { react: { version: "detect" } },
+		rules: {
+			...pluginReactHooks.configs.recommended.rules,
+			// React scope no longer necessary with new JSX transform.
+			"react/react-in-jsx-scope": "off",
+		},
+	},
+]

evals/config/eslint/package.json

@@ -0,0 +1,22 @@
+{
+	"name": "@evals/eslint-config",
+	"private": true,
+	"type": "module",
+	"exports": {
+		"./base": "./base.js",
+		"./next-js": "./next.js"
+	},
+	"devDependencies": {
+		"@eslint/js": "^9.22.0",
+		"@next/eslint-plugin-next": "^15.2.1",
+		"eslint": "^9.22.0",
+		"eslint-config-prettier": "^10.1.1",
+		"eslint-plugin-only-warn": "^1.1.0",
+		"eslint-plugin-react": "^7.37.4",
+		"eslint-plugin-react-hooks": "^5.2.0",
+		"eslint-plugin-turbo": "^2.4.4",
+		"globals": "^16.0.0",
+		"typescript": "^5",
+		"typescript-eslint": "^8.26.0"
+	}
+}

evals/config/typescript/base.json

@@ -0,0 +1,19 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"compilerOptions": {
+		"declaration": true,
+		"declarationMap": true,
+		"esModuleInterop": true,
+		"incremental": false,
+		"isolatedModules": true,
+		"lib": ["es2022", "DOM", "DOM.Iterable"],
+		"module": "NodeNext",
+		"moduleDetection": "force",
+		"moduleResolution": "NodeNext",
+		"noUncheckedIndexedAccess": true,
+		"resolveJsonModule": true,
+		"skipLibCheck": true,
+		"strict": true,
+		"target": "ES2022"
+	}
+}

benchmark/tsconfig.json → evals/config/typescript/cjs.json

@@ -1,4 +1,5 @@
 {
+	"$schema": "https://json.schemastore.org/tsconfig",
 	"compilerOptions": {
 		"module": "CommonJS",
 		"moduleResolution": "Node",
@@ -8,9 +9,6 @@
 		"sourceMap": true,
 		"strict": true,
 		"skipLibCheck": true,
-		"useUnknownInCatchVariables": false,
-		"outDir": "out"
-	},
-	"include": ["src", "../src/exports/roo-code.d.ts"],
-	"exclude": ["**/node_modules/**", "out"]
+		"useUnknownInCatchVariables": false
+	}
 }

evals/config/typescript/nextjs.json

@@ -0,0 +1,12 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"extends": "./base.json",
+	"compilerOptions": {
+		"plugins": [{ "name": "next" }],
+		"module": "ESNext",
+		"moduleResolution": "bundler",
+		"allowJs": true,
+		"jsx": "preserve",
+		"noEmit": true
+	}
+}

evals/config/typescript/package.json

@@ -0,0 +1,7 @@
+{
+	"name": "@evals/typescript-config",
+	"private": true,
+	"publishConfig": {
+		"access": "public"
+	}
+}

evals/package.json

@@ -0,0 +1,26 @@
+{
+	"name": "@evals/monorepo",
+	"private": true,
+	"packageManager": "[email protected]+sha512.2d92c86b7928dc8284f53494fb4201f983da65f0fb4f0d40baafa5cf628fa31dae3e5968f12466f17df7e97310e30f343a648baea1b9b350685dafafffdf5808",
+	"scripts": {
+		"lint": "turbo lint --log-order grouped --output-logs new-only",
+		"check-types": "turbo check-types --log-order grouped --output-logs new-only",
+		"test": "turbo test --log-order grouped --output-logs new-only",
+		"format": "turbo format --log-order grouped --output-logs new-only",
+		"build": "turbo build --log-order grouped --output-logs new-only",
+		"web": "turbo dev --filter @evals/web",
+		"cli": "turbo dev --filter @evals/cli -- run",
+		"drizzle:studio": "pnpm --filter @evals/db db:studio"
+	},
+	"devDependencies": {
+		"@dotenvx/dotenvx": "^1.39.0",
+		"@eslint/js": "^9.22.0",
+		"eslint": "^9.22.0",
+		"globals": "^16.0.0",
+		"prettier": "^3.5.3",
+		"tsx": "^4.19.3",
+		"turbo": "^2.4.4",
+		"typescript": "^5",
+		"typescript-eslint": "^8.26.0"
+	}
+}

evals/packages/db/README.md

@@ -0,0 +1,15 @@
+## Running Migrations
+
+Update `src/schema.ts` as needed, and then run:
+
+```sh
+pnpm db:generate
+```
+
+Inspect the generated sql in the migration filed added to `drizzle/`.
+
+If it looks okay, then run:
+
+```sh
+pnpm db:migrate
+```

evals/packages/db/drizzle.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "drizzle-kit"
+
+export default defineConfig({
+	out: "./drizzle",
+	schema: "./src/schema.ts",
+	dialect: "sqlite",
+	dbCredentials: {
+		url: process.env.BENCHMARKS_DB_PATH!,
+	},
+})