Add sitemap + robots.txt to leaderboard.querygym.com

reproducibility/site/astro.config.mjs

@@ -1,11 +1,33 @@
 import { defineConfig } from "astro/config";
 import tailwind from "@astrojs/tailwind";
 import solid from "@astrojs/solid-js";
+import sitemap from "@astrojs/sitemap";
 
 export default defineConfig({
   site: "https://leaderboard.querygym.com",
   output: "static",
-  integrations: [tailwind({ applyBaseStyles: false }), solid()],
+  integrations: [
+    tailwind({ applyBaseStyles: false }),
+    solid(),
+    sitemap({
+      changefreq: "weekly",
+      priority: 0.5,
+      lastmod: new Date(),
+      serialize(item) {
+        // Home page is the canonical entry point.
+        if (item.url === "https://leaderboard.querygym.com/") {
+          return { ...item, priority: 1.0, changefreq: "weekly" };
+        }
+        // Per-run detail pages are mostly internal-link targets — give them
+        // lower priority so search engines focus on the dataset/method/model
+        // index pages.
+        if (item.url.startsWith("https://leaderboard.querygym.com/runs/")) {
+          return { ...item, priority: 0.3, changefreq: "monthly" };
+        }
+        return item;
+      },
+    }),
+  ],
   vite: {
     ssr: {
       // Needed because we read CSV/YAML at build time from outside the project root.

reproducibility/site/package.json

@@ -13,22 +13,23 @@
     "check": "astro check"
   },
   "dependencies": {
-    "@qg/shared": "workspace:*",
-    "astro": "^5.0.0",
-    "@astrojs/tailwind": "^5.1.5",
+    "@astrojs/sitemap": "^3.7.2",
     "@astrojs/solid-js": "^5.0.0",
-    "solid-js": "^1.9.0",
+    "@astrojs/tailwind": "^5.1.5",
+    "@qg/shared": "workspace:*",
     "@tanstack/solid-table": "^8.20.0",
-    "tailwindcss": "^3.4.0",
+    "astro": "^5.0.0",
+    "js-yaml": "^4.1.0",
     "papaparse": "^5.4.1",
-    "js-yaml": "^4.1.0"
+    "solid-js": "^1.9.0",
+    "tailwindcss": "^3.4.0"
   },
   "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^20.0.0",
+    "@types/papaparse": "^5.3.15",
     "tsx": "^4.19.0",
     "typescript": "^5.6.0",
-    "wrangler": "^4.0.0",
-    "@types/papaparse": "^5.3.15",
-    "@types/js-yaml": "^4.0.9",
-    "@types/node": "^20.0.0"
+    "wrangler": "^4.0.0"
   }
 }

reproducibility/site/public/robots.txt

@@ -0,0 +1,5 @@
+# leaderboard.querygym.com — open to all well-behaved crawlers.
+User-agent: *
+Allow: /
+
+Sitemap: https://leaderboard.querygym.com/sitemap-index.xml

reproducibility/site/src/layouts/Default.astro

@@ -29,6 +29,7 @@ const navLinks = [
     <title>{title} — QueryGym Leaderboard</title>
     {description && <meta name="description" content={description} />}
     <link rel="icon" type="image/png" href="/querygym-logo.png" />
+    <link rel="sitemap" type="application/xml" title="Sitemap" href="/sitemap-index.xml" />
 
     <GoogleAnalytics />
   </head>