feat: content quality scoring, claude code config command, eval improvements

Author: jeff-vincent

.gitignore

@@ -8,3 +8,4 @@ models/*.gguf
 website/node_modules/
 website/build/
 website/.docusaurus/
+eval-bin

cmd/memoryd/claude.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+// registerMCPServers attempts to register memoryd as an MCP server in every
+// supported coding agent found on this machine. Each registration is
+// idempotent and non-fatal — failures are logged and skipped.
+func registerMCPServers() {
+	execPath, err := os.Executable()
+	if err != nil {
+		log.Printf("[mcp-register] could not determine executable path: %v", err)
+		return
+	}
+
+	home, err := os.UserHomeDir()
+	if err != nil {
+		log.Printf("[mcp-register] could not determine home directory: %v", err)
+		return
+	}
+
+	entry := mcpEntry(execPath)
+
+	registrars := []struct {
+		name string
+		fn   func() error
+	}{
+		{"Claude Code", func() error { return registerClaudeCode(home, entry) }},
+		{"Cursor", func() error { return registerStandard(home, cursorMCPPath(home), entry) }},
+		{"Windsurf", func() error { return registerStandard(home, windsurfMCPPath(home), entry) }},
+		{"Cline", func() error { return registerStandard(home, clineMCPPath(home), entry) }},
+	}
+
+	for _, r := range registrars {
+		if err := r.fn(); err != nil {
+			log.Printf("[mcp-register] %s: %v", r.name, err)
+		}
+	}
+}
+
+// mcpEntry returns the standard MCP server entry for memoryd.
+func mcpEntry(execPath string) map[string]any {
+	return map[string]any{
+		"command": execPath,
+		"args":    []string{"mcp"},
+		"env":     map[string]any{},
+	}
+}
+
+// --- Claude Code ---
+
+// registerClaudeCode writes to ~/.claude/settings.json under
+// projects[home].mcpServers. Created if it doesn't exist.
+func registerClaudeCode(home string, entry map[string]any) error {
+	settingsPath := filepath.Join(home, ".claude", "settings.json")
+
+	var settings map[string]any
+	data, err := os.ReadFile(settingsPath)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &settings); err != nil {
+			return err
+		}
+	}
+	if settings == nil {
+		settings = map[string]any{}
+	}
+
+	projects, _ := settings["projects"].(map[string]any)
+	if projects == nil {
+		projects = map[string]any{}
+		settings["projects"] = projects
+	}
+
+	homeProject, _ := projects[home].(map[string]any)
+	if homeProject == nil {
+		homeProject = map[string]any{}
+		projects[home] = homeProject
+	}
+
+	mcpServers, _ := homeProject["mcpServers"].(map[string]any)
+	if mcpServers == nil {
+		mcpServers = map[string]any{}
+		homeProject["mcpServers"] = mcpServers
+	}
+
+	if _, exists := mcpServers["memoryd"]; exists {
+		return nil
+	}
+
+	mcpServers["memoryd"] = entry
+
+	return writeJSON(settingsPath, settings)
+}
+
+// --- Standard format (Cursor, Windsurf, Cline) ---
+
+// registerStandard writes an {"mcpServers":{"memoryd":{...}}} config file.
+// It only proceeds if the agent's config directory already exists (proving
+// the agent is installed). The config file itself is created if absent.
+func registerStandard(home, configPath string, entry map[string]any) error {
+	dir := filepath.Dir(configPath)
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		return nil // agent not installed, skip silently
+	}
+
+	var cfg map[string]any
+	data, err := os.ReadFile(configPath)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	if len(data) > 0 {
+		if err := json.Unmarshal(data, &cfg); err != nil {
+			return err
+		}
+	}
+	if cfg == nil {
+		cfg = map[string]any{}
+	}
+
+	mcpServers, _ := cfg["mcpServers"].(map[string]any)
+	if mcpServers == nil {
+		mcpServers = map[string]any{}
+		cfg["mcpServers"] = mcpServers
+	}
+
+	if _, exists := mcpServers["memoryd"]; exists {
+		return nil
+	}
+
+	mcpServers["memoryd"] = entry
+
+	if err := writeJSON(configPath, cfg); err != nil {
+		return err
+	}
+
+	log.Printf("[mcp-register] registered in %s", configPath)
+	return nil
+}
+
+// --- Path helpers ---
+
+func cursorMCPPath(home string) string {
+	return filepath.Join(home, ".cursor", "mcp.json")
+}
+
+func windsurfMCPPath(home string) string {
+	return filepath.Join(home, ".codeium", "windsurf", "mcp_config.json")
+}
+
+func clineMCPPath(home string) string {
+	switch runtime.GOOS {
+	case "darwin":
+		return filepath.Join(home, "Library", "Application Support", "Code", "User",
+			"globalStorage", "saoudrizwan.claude-dev", "settings", "cline_mcp_settings.json")
+	default:
+		return filepath.Join(home, ".config", "Code", "User",
+			"globalStorage", "saoudrizwan.claude-dev", "settings", "cline_mcp_settings.json")
+	}
+}
+
+// --- Helpers ---
+
+func writeJSON(path string, v any) error {
+	out, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+	return os.WriteFile(path, out, 0644)
+}

cmd/memoryd/main.go

@@ -74,6 +74,8 @@ func startCmd() *cobra.Command {
 				return err
 			}
 
+			registerMCPServers()
+
 			if cfg.MongoDBAtlasURI == "" {
 				return fmt.Errorf("mongodb_atlas_uri is required -- edit %s", config.Path())
 			}
@@ -161,7 +163,12 @@ func startCmd() *cobra.Command {
 			// Write pipeline uses primary store (default write target).
 			qt := quality.NewTracker(primary.Mongo, quality.DefaultThreshold)
 			read := pipeline.NewReadPipeline(emb, multi, cfg, pipeline.WithQualityTracker(qt))
-			write := pipeline.NewWritePipeline(emb, primary.Store)
+
+			scorer, err := quality.NewContentScorer(ctx, emb)
+			if err != nil {
+				log.Printf("warning: content scorer unavailable, chunks will not be quality-scored: %v", err)
+			}
+			write := pipeline.NewWritePipeline(emb, primary.Store, pipeline.WithContentScorer(scorer))
 
 			// 4. Build ingester (operates on primary database)
 			ing := ingest.NewIngester(emb, primary.Mongo, primary.Mongo)

eval/harness.go

@@ -24,10 +24,11 @@ type Scenario struct {
 }
 
 type RunResult struct {
-	Condition string        `json:"condition"`
-	Response  string        `json:"response"`
-	Latency   time.Duration `json:"latency_ms"`
-	Context   string        `json:"context,omitempty"`
+	Condition       string    `json:"condition"`
+	Response        string    `json:"response"`
+	Latency         time.Duration `json:"latency_ms"`
+	Context         string    `json:"context,omitempty"`
+	RetrievalScores []float64 `json:"retrieval_scores,omitempty"`
 }
 
 type JudgeScore struct {
@@ -169,22 +170,23 @@ func (h *Harness) clearMemories(ctx context.Context) error {
 	return nil
 }
 
-func (h *Harness) retrieveContext(ctx context.Context, query string) (string, error) {
+func (h *Harness) retrieveContext(ctx context.Context, query string) (string, []float64, error) {
 	body, _ := json.Marshal(map[string]string{"query": query})
 	req, _ := http.NewRequestWithContext(ctx, "POST", h.cfg.MemorydURL+"/api/search", bytes.NewReader(body))
 	req.Header.Set("Content-Type", "application/json")
 	resp, err := h.client.Do(req)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
 	defer resp.Body.Close()
 	var result struct {
-		Context string `json:"context"`
+		Context string    `json:"context"`
+		Scores  []float64 `json:"scores"`
 	}
 	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-		return "", err
+		return "", nil, err
 	}
-	return result.Context, nil
+	return result.Context, result.Scores, nil
 }
 
 func (h *Harness) callClaude(ctx context.Context, system, user string) (string, time.Duration, error) {
@@ -237,7 +239,7 @@ func (h *Harness) runBare(ctx context.Context, sc Scenario) (*RunResult, error)
 }
 
 func (h *Harness) runAugmented(ctx context.Context, sc Scenario) (*RunResult, error) {
-	retrieved, err := h.retrieveContext(ctx, sc.Prompt)
+	retrieved, scores, err := h.retrieveContext(ctx, sc.Prompt)
 	if err != nil {
 		return nil, fmt.Errorf("retrieve: %w", err)
 	}
@@ -249,7 +251,7 @@ func (h *Harness) runAugmented(ctx context.Context, sc Scenario) (*RunResult, er
 	if err != nil {
 		return nil, err
 	}
-	return &RunResult{Condition: "augmented", Response: text, Latency: latency, Context: retrieved}, nil
+	return &RunResult{Condition: "augmented", Response: text, Latency: latency, Context: retrieved, RetrievalScores: scores}, nil
 }
 
 func (h *Harness) judge(ctx context.Context, sc Scenario, bare, aug *RunResult) ([]JudgeScore, error) {

eval/report.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math"
 	"strings"
 )
 
@@ -30,6 +31,22 @@ func Report(w io.Writer, results []ScenarioResult) {
 			fmt.Fprintf(w, "      %s\n", s.Explanation)
 		}
 
+		if len(r.Augmented.RetrievalScores) > 0 {
+			min, max, avg := scoreStats(r.Augmented.RetrievalScores)
+			fmt.Fprintf(w, "\n  retrieval: n=%d  min=%.3f  max=%.3f  avg=%.3f",
+				len(r.Augmented.RetrievalScores), min, max, avg)
+			if r.Delta > 0 {
+				fmt.Fprintf(w, "  → helped")
+			} else if r.Delta < 0 {
+				fmt.Fprintf(w, "  → hurt")
+			} else {
+				fmt.Fprintf(w, "  → neutral")
+			}
+			fmt.Fprintf(w, "\n")
+		} else {
+			fmt.Fprintf(w, "\n  retrieval: no scores recorded\n")
+		}
+
 		fmt.Fprintf(w, "\n  TOTAL: bare=%d  aug=%d  delta=%+d\n\n", r.BareTotal, r.AugTotal, r.Delta)
 
 		totalBare += r.BareTotal
@@ -46,9 +63,71 @@ func Report(w io.Writer, results []ScenarioResult) {
 		pct := float64(totalAug-totalBare) / float64(totalBare) * 100
 		fmt.Fprintf(w, "    Improvement:     %.1f%%\n", pct)
 	}
+
+	// Score-vs-delta breakdown: bucket by avg retrieval score, show avg delta per bucket.
+	fmt.Fprintf(w, "\n  Retrieval score vs quality delta:\n")
+	type bucket struct {
+		count    int
+		deltaSum int
+	}
+	buckets := map[string]*bucket{
+		"<0.50": {},
+		"0.50-0.60": {},
+		"0.60-0.70": {},
+		"0.70-0.80": {},
+		">=0.80": {},
+	}
+	bucketOrder := []string{"<0.50", "0.50-0.60", "0.60-0.70", "0.70-0.80", ">=0.80"}
+	for _, r := range results {
+		if len(r.Augmented.RetrievalScores) == 0 {
+			continue
+		}
+		_, _, avg := scoreStats(r.Augmented.RetrievalScores)
+		var key string
+		switch {
+		case avg < 0.50:
+			key = "<0.50"
+		case avg < 0.60:
+			key = "0.50-0.60"
+		case avg < 0.70:
+			key = "0.60-0.70"
+		case avg < 0.80:
+			key = "0.70-0.80"
+		default:
+			key = ">=0.80"
+		}
+		buckets[key].count++
+		buckets[key].deltaSum += r.Delta
+	}
+	for _, k := range bucketOrder {
+		b := buckets[k]
+		if b.count == 0 {
+			continue
+		}
+		avgDelta := float64(b.deltaSum) / float64(b.count)
+		fmt.Fprintf(w, "    avg score %-12s  n=%d  avg delta=%+.2f\n", k, b.count, avgDelta)
+	}
+
 	fmt.Fprintf(w, "%s\n\n", strings.Repeat("=", 72))
 }
 
+func scoreStats(scores []float64) (min, max, avg float64) {
+	min = math.MaxFloat64
+	max = -math.MaxFloat64
+	var sum float64
+	for _, s := range scores {
+		if s < min {
+			min = s
+		}
+		if s > max {
+			max = s
+		}
+		sum += s
+	}
+	avg = sum / float64(len(scores))
+	return
+}
+
 // ReportJSON writes machine-readable JSON output.
 func ReportJSON(w io.Writer, results []ScenarioResult) error {
 	enc := json.NewEncoder(w)

internal/config/config.go

@@ -25,10 +25,10 @@ const (
 
 // DatabaseConfig describes a single team database connection.
 type DatabaseConfig struct {
-	Name     string `yaml:"name"`               // Human label (e.g., "platform", "payments")
-	Database string `yaml:"database"`           // MongoDB database name
-	Role     string `yaml:"role"`               // "full" or "read-only"
-	URI      string `yaml:"uri,omitempty"`      // Connection string (empty = use primary URI)
+	Name     string `yaml:"name"`              // Human label (e.g., "platform", "payments")
+	Database string `yaml:"database"`          // MongoDB database name
+	Role     string `yaml:"role"`              // "full" or "read-only"
+	URI      string `yaml:"uri,omitempty"`     // Connection string (empty = use primary URI)
 	Enabled  *bool  `yaml:"enabled,omitempty"` // nil/true = enabled, false = disabled
 }