feat: implement sitemap generation in TypeScript and remove XML file (#6206)

Co-authored-by: daniel-lxs <[email protected]>
Co-authored-by: Daniel <[email protected]>

apps/web-roo-code/.env.example

@@ -6,3 +6,7 @@ NEXT_PUBLIC_POSTHOG_HOST=https://us.i.posthog.com
 # Basin Form Endpoint for Static Form Submissions
 # Replace this with your actual Basin form endpoint (e.g., https://usebasin.com/f/your-form-id)
 NEXT_PUBLIC_BASIN_ENDPOINT=https://usebasin.com/f/your-form-id-here
+
+# Site URL Configuration
+# Used for generating absolute URLs in sitemap and other contexts
+NEXT_PUBLIC_SITE_URL=https://roocode.com

apps/web-roo-code/.gitignore

@@ -40,3 +40,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# generated files
+/public/sitemap*.xml
+/public/robots.txt

apps/web-roo-code/.npmrc

@@ -0,0 +1 @@
+enable-pre-post-scripts=true

apps/web-roo-code/next-sitemap.config.cjs

@@ -0,0 +1,57 @@
+/** @type {import('next-sitemap').IConfig} */
+module.exports = {
+  siteUrl: process.env.NEXT_PUBLIC_SITE_URL || 'https://roocode.com',
+  generateRobotsTxt: true,
+  generateIndexSitemap: false, // We don't need index sitemap for a small site
+  changefreq: 'monthly',
+  priority: 0.7,
+  sitemapSize: 5000,
+  exclude: [
+    '/api/*',
+    '/server-sitemap-index.xml',
+    '/404',
+    '/500',
+    '/_not-found',
+  ],
+  robotsTxtOptions: {
+    policies: [
+      {
+        userAgent: '*',
+        allow: '/',
+      },
+    ],
+    additionalSitemaps: [
+      // Add any additional sitemaps here if needed in the future
+    ],
+  },
+  // Custom transform function to set specific priorities and change frequencies
+  transform: async (config, path) => {
+    // Set custom priority for specific pages
+    let priority = config.priority;
+    let changefreq = config.changefreq;
+    
+    if (path === '/') {
+      priority = 1.0;
+      changefreq = 'yearly';
+    } else if (path === '/enterprise' || path === '/evals') {
+      priority = 0.8;
+      changefreq = 'monthly';
+    } else if (path === '/privacy' || path === '/terms') {
+      priority = 0.5;
+      changefreq = 'yearly';
+    }
+    
+    return {
+      loc: path,
+      changefreq,
+      priority,
+      lastmod: config.autoLastmod ? new Date().toISOString() : undefined,
+      alternateRefs: config.alternateRefs ?? [],
+    };
+  },
+  additionalPaths: async (config) => {
+    // Add any additional paths that might not be automatically discovered
+    // This is useful for dynamic routes or API-generated pages
+    return [];
+  },
+};

apps/web-roo-code/package.json

@@ -7,6 +7,7 @@
 		"check-types": "tsc --noEmit",
 		"dev": "next dev",
 		"build": "next build",
+		"postbuild": "next-sitemap --config next-sitemap.config.cjs",
 		"start": "next start",
 		"clean": "rimraf .next .turbo"
 	},
@@ -42,6 +43,7 @@
 		"@types/react": "^18.3.23",
 		"@types/react-dom": "^18.3.7",
 		"autoprefixer": "^10.4.21",
+		"next-sitemap": "^4.2.3",
 		"postcss": "^8.5.4",
 		"tailwindcss": "^3.4.17"
 	}

apps/web-roo-code/src/app/sitemap.xml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-  <url>
-    <loc>https://roocode.com/</loc>
-    <lastmod>2025-03-13T22:26:09Z</lastmod>
-    <changefreq>yearly</changefreq>
-    <priority>1.0</priority>
-  </url>
-</urlset>

knip.json

@@ -7,7 +7,8 @@
 		"src/activate/**",
 		"src/workers/countTokens.ts",
 		"src/extension.ts",
-		"scripts/**"
+		"scripts/**",
+		"apps/web-roo-code/next-sitemap.config.cjs"
 	],
 	"workspaces": {
 		"src": {

pnpm-lock.yaml

@@ -342,6 +342,9 @@ importers:
       autoprefixer:
         specifier: ^10.4.21
         version: 10.4.21([email protected])
+      next-sitemap:
+        specifier: ^4.2.3
+        version: 4.2.3([email protected]([email protected]([email protected]))([email protected]))
       postcss:
         specifier: ^8.5.4
         version: 8.5.4
@@ -1532,6 +1535,9 @@ packages:
   '@chevrotain/[email protected]':
     resolution: {integrity: sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==}
 
+  '@corex/[email protected]':
+    resolution: {integrity: sha512-N8uEMrMPL0cu/bdboEWpQYb/0i2K5Qn8eCsxzOmxSggJbbQte7ljMRoXm917AbntqTGOzdTu+vP3KOOzoC70HQ==}
+
   '@csstools/[email protected]':
     resolution: {integrity: sha512-JqWH1vsgdGcw2RR6VliXXdA0/59LttzlU8UlRT/iUUsEeWfYq8I+K0yhihEUTTHLRm1EXvpsCx3083EU15ecsA==}
     engines: {node: '>=18'}
@@ -2132,6 +2138,9 @@ packages:
   '@neon-rs/[email protected]':
     resolution: {integrity: sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==}
 
+  '@next/[email protected]':
+    resolution: {integrity: sha512-fbb2C7HChgM7CemdCY+y3N1n8pcTKdqtQLbC7/EQtPdLvlMUT9JX/dBYl8MMZAtYG4uVMyPFHXckb68q/NRwqg==}
+
   '@next/[email protected]':
     resolution: {integrity: sha512-uWkCf9C8wKTyQjqrNk+BA7eL3LOQdhL+xlmJUf2O85RM4lbzwBwot3Sqv2QGe/RGnc3zysIf1oJdtq9S00pkmQ==}
 
@@ -7485,6 +7494,13 @@ packages:
     resolution: {integrity: sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==}
     engines: {node: '>= 0.4.0'}
 
+  [email protected]:
+    resolution: {integrity: sha512-vjdCxeDuWDzldhCnyFCQipw5bfpl4HmZA7uoo3GAaYGjGgfL4Cxb1CiztPuWGmS+auYs7/8OekRS8C2cjdAsjQ==}
+    engines: {node: '>=14.18'}
+    hasBin: true
+    peerDependencies:
+      next: '*'
+
   [email protected]:
     resolution: {integrity: sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA==}
     peerDependencies:
@@ -10821,6 +10837,8 @@ snapshots:
 
   '@chevrotain/[email protected]': {}
 
+  '@corex/[email protected]': {}
+
   '@csstools/[email protected]': {}
 
   '@csstools/[email protected](@csstools/[email protected](@csstools/[email protected]))(@csstools/[email protected])':
@@ -11418,6 +11436,8 @@ snapshots:
   '@neon-rs/[email protected]':
     optional: true
 
+  '@next/[email protected]': {}
+
   '@next/[email protected]': {}
 
   '@next/[email protected]':
@@ -17502,6 +17522,14 @@ snapshots:
 
   [email protected]: {}
 
+  [email protected]([email protected]([email protected]([email protected]))([email protected])):
+    dependencies:
+      '@corex/deepmerge': 4.0.43
+      '@next/env': 13.5.11
+      fast-glob: 3.3.3
+      minimist: 1.2.8
+      next: 15.2.5([email protected]([email protected]))([email protected])
+
   [email protected]([email protected]([email protected]))([email protected]):
     dependencies:
       react: 18.3.1