feat: Support .env files by default via dotenv/config

.changeset/0234-auto-dotenv-support.md

@@ -0,0 +1,5 @@
+---
+"evalite": minor
+---
+
+Support .env files by default via dotenv/config. Environment variables from .env files are now automatically loaded without any configuration needed. Users no longer need to manually add `setupFiles: ["dotenv/config"]` to their evalite.config.ts.

apps/evalite-docs/src/content/docs/guides/environment-variables.mdx

@@ -8,6 +8,8 @@ To call your LLM from a third-party service, you'll likely need some environment
 
 ## Setting Up Env Variables
 
+Evalite automatically loads environment variables from `.env` files in your project root. No configuration needed!
+
 <Steps>
 
 1. Create a `.env` file in the root of your project:
@@ -24,24 +26,17 @@ To call your LLM from a third-party service, you'll likely need some environment
    .env
    ```
 
-3. Install `dotenv`:
-
-   ```bash
-   pnpm add -D dotenv
-   ```
-
-4. Add an `evalite.config.ts` file:
+</Steps>
 
-   ```ts
-   // evalite.config.ts
+Now, your environment variables will be available in your evals.
 
-   import { defineConfig } from "evalite/config";
+## How It Works
 
-   export default defineConfig({
-     setupFiles: ["dotenv/config"],
-   });
-   ```
+Evalite uses [dotenv](https://www.npmjs.com/package/dotenv) under the hood to automatically load environment variables from `.env` files. The following files are supported (in order of precedence):
 
-</Steps>
+- `.env.local`
+- `.env.[mode].local`
+- `.env.[mode]`
+- `.env`
 
-Now, your environment variables will be available in your evals.
+This happens automatically—no setup required!

packages/evalite-tests/tests/config.test.ts

@@ -50,3 +50,35 @@ it("setupFiles in evalite.config.ts should load environment variables", async ()
     "test_value_from_env"
   );
 });
+
+it("setupFiles in vitest.config.ts should be supported", async () => {
+  await using fixture = await loadFixture("config-setupfiles-vitest");
+
+  await fixture.run({
+    mode: "run-once-and-exit",
+  });
+
+  const evals = await getEvalsAsRecordViaStorage(fixture.storage);
+
+  // Should complete successfully with env var loaded from vitest.config.ts
+  expect(evals["Vitest Setup Test"]).toHaveLength(1);
+  expect(evals["Vitest Setup Test"]?.[0]?.status).toBe("success");
+  expect(evals["Vitest Setup Test"]?.[0]?.results[0]?.output).toBe(
+    "from_vitest_config"
+  );
+});
+
+it("setupFiles in evalite.config.ts should take precedence over vitest.config.ts", async () => {
+  await using fixture = await loadFixture("config-setupfiles-precedence");
+
+  await fixture.run({
+    mode: "run-once-and-exit",
+  });
+
+  const evals = await getEvalsAsRecordViaStorage(fixture.storage);
+
+  // Should complete successfully with env var from evalite setup (which runs after vitest)
+  expect(evals["Precedence Test"]).toHaveLength(1);
+  expect(evals["Precedence Test"]?.[0]?.status).toBe("success");
+  expect(evals["Precedence Test"]?.[0]?.results[0]?.output).toBe("evalite");
+});

packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite-setup.ts

@@ -0,0 +1,2 @@
+// Setup file from evalite.config.ts - should override vitest
+process.env.SETUP_ORDER = "evalite";

packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/evalite.config.ts

@@ -0,0 +1,5 @@
+import { defineConfig } from "evalite/config";
+
+export default defineConfig({
+  setupFiles: ["./evalite-setup.ts"],
+});

packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/test.eval.ts

@@ -0,0 +1,15 @@
+import { evalite } from "evalite";
+import { Levenshtein } from "autoevals";
+
+evalite("Precedence Test", {
+  data: () => [
+    {
+      input: "test",
+      expected: "evalite",
+    },
+  ],
+  task: async (input) => {
+    return process.env.SETUP_ORDER as string;
+  },
+  scorers: [Levenshtein],
+});

packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest-setup.ts

@@ -0,0 +1,2 @@
+// Setup file from vitest.config.ts
+process.env.SETUP_ORDER = "vitest";

packages/evalite-tests/tests/fixtures/config-setupfiles-precedence/vitest.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    setupFiles: ["./vitest-setup.ts"],
+  },
+});

packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/test.eval.ts

@@ -0,0 +1,15 @@
+import { evalite } from "evalite";
+import { Levenshtein } from "autoevals";
+
+evalite("Vitest Setup Test", {
+  data: () => [
+    {
+      input: "test",
+      expected: process.env.VITEST_SETUP_VAR,
+    },
+  ],
+  task: async (input) => {
+    return process.env.VITEST_SETUP_VAR as string;
+  },
+  scorers: [Levenshtein],
+});

packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest-setup.ts

@@ -0,0 +1,2 @@
+// Setup file from vitest.config.ts
+process.env.VITEST_SETUP_VAR = "from_vitest_config";

packages/evalite-tests/tests/fixtures/config-setupfiles-vitest/vitest.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    setupFiles: ["./vitest-setup.ts"],
+  },
+});

packages/evalite/package.json

@@ -45,7 +45,8 @@
     "./runner": "./dist/run-evalite.js",
     "./export-static": "./dist/export-static.js",
     "./sqlite-storage": "./dist/storage/sqlite.js",
-    "./in-memory-storage": "./dist/storage/in-memory.js"
+    "./in-memory-storage": "./dist/storage/in-memory.js",
+    "./env-setup-file": "./dist/env-setup-file.js"
   },
   "dependencies": {
     "@ai-sdk/provider": "^2.0.0",
@@ -55,6 +56,7 @@
     "@stricli/core": "^1.2.0",
     "@vitest/runner": "^3.2.4",
     "better-sqlite3": "^11.6.0",
+    "dotenv": "^16.4.7",
     "fastify": "^5.6.1",
     "file-type": "^19.6.0",
     "jiti": "^2.6.1",

packages/evalite/src/config.ts

@@ -31,6 +31,13 @@ const CONFIG_FILE_NAMES = [
   "evalite.config.mjs",
 ];
 
+const VITEST_CONFIG_FILE_NAMES = [
+  "vitest.config.ts",
+  "vitest.config.mts",
+  "vitest.config.js",
+  "vitest.config.mjs",
+];
+
 /**
  * Load Evalite configuration file from the specified directory.
  * Looks for evalite.config.{ts,mts,js,mjs} files.
@@ -74,3 +81,49 @@ export async function loadEvaliteConfig(
 
   return undefined;
 }
+
+/**
+ * Load Vitest configuration file from the specified directory.
+ * Looks for vitest.config.{ts,mts,js,mjs} files and extracts setupFiles.
+ *
+ * @param cwd - Current working directory to search for config file
+ * @returns Array of setupFiles from vitest config, or empty array if none found
+ */
+export async function loadVitestSetupFiles(
+  cwd: string
+): Promise<string[]> {
+  const jiti = createJiti(import.meta.url, {
+    interopDefault: true,
+    requireCache: false,
+  });
+
+  for (const fileName of VITEST_CONFIG_FILE_NAMES) {
+    const configPath = path.join(cwd, fileName);
+
+    try {
+      const loaded = (await jiti.import(configPath)) as any;
+      const config = loaded.default || loaded;
+
+      if (config && typeof config === "object" && config.test?.setupFiles) {
+        const setupFiles = config.test.setupFiles;
+        // setupFiles can be a string or array of strings
+        return Array.isArray(setupFiles) ? setupFiles : [setupFiles];
+      }
+    } catch (error: any) {
+      // File not found is expected, ignore it
+      if (
+        error.code === "ERR_MODULE_NOT_FOUND" ||
+        error.code === "ENOENT" ||
+        error.message?.includes("Cannot find module")
+      ) {
+        continue;
+      }
+      // Other errors (syntax errors, etc) should be thrown
+      throw new Error(
+        `Failed to load Vitest config from ${configPath}: ${error.message}`
+      );
+    }
+  }
+
+  return [];
+}

packages/evalite/src/env-setup-file.ts

@@ -0,0 +1,3 @@
+// This file is automatically loaded by Evalite to support .env files
+// It loads environment variables from .env files using dotenv
+import "dotenv/config";

packages/evalite/src/run-evalite.ts

@@ -10,7 +10,7 @@ import EvaliteReporter from "./reporter.js";
 import { createServer } from "./server.js";
 import type { Evalite } from "./types.js";
 import { createSqliteStorage } from "./storage/sqlite.js";
-import { loadEvaliteConfig } from "./config.js";
+import { loadEvaliteConfig, loadVitestSetupFiles } from "./config.js";
 
 declare module "vitest" {
   export interface ProvidedContext {
@@ -208,6 +208,9 @@ export const runEvalite = async (opts: {
   // Load config file if present
   const config = await loadEvaliteConfig(cwd);
 
+  // Load setupFiles from vitest.config.ts
+  const vitestSetupFiles = await loadVitestSetupFiles(cwd);
+
   // Merge options: opts (highest priority) > config > defaults
   let storage = opts.storage;
 
@@ -226,7 +229,16 @@ export const runEvalite = async (opts: {
   const serverPort = config?.server?.port ?? DEFAULT_SERVER_PORT;
   const testTimeout = config?.testTimeout;
   const maxConcurrency = config?.maxConcurrency;
-  const setupFiles = config?.setupFiles;
+
+  // Merge setupFiles:
+  // 1. Always include env-setup-file first to load .env files
+  // 2. Add setupFiles from vitest.config.ts
+  // 3. Add setupFiles from evalite.config.ts (takes precedence)
+  const setupFiles = [
+    "evalite/env-setup-file",
+    ...vitestSetupFiles,
+    ...(config?.setupFiles || []),
+  ];
 
   const filters = opts.path ? [opts.path] : undefined;
   process.env.EVALITE_REPORT_TRACES = "true";

packages/evalite/src/types.ts

@@ -85,11 +85,12 @@ export declare namespace Evalite {
     trialCount?: number;
 
     /**
-     * Setup files to run before tests (e.g., for loading environment variables)
+     * Setup files to run before tests.
+     * Note: .env files are loaded automatically via dotenv - no need to configure.
      * @example
      * ```ts
      * export default defineConfig({
-     *   setupFiles: ["dotenv/config"]
+     *   setupFiles: ["./custom-setup.ts"]
      * })
      * ```
      */

packages/example/evalite.config.ts

@@ -1,5 +1,5 @@
 import { defineConfig } from "evalite/config";
 
 export default defineConfig({
-  setupFiles: ["dotenv/config"],
+  // .env files are now loaded automatically!
 });