integration_test.go

package tok_test

import (
	"strings"
	"testing"
	"time"

	"github.com/GrayCodeAI/tok"
)

// ---------------------------------------------------------------------------
// 1. Full Pipeline - End-to-end compression of text through all stages
// ---------------------------------------------------------------------------

func TestIntegration_FullPipeline_Minimal(t *testing.T) {
	input := `This is a fairly long piece of text that contains multiple sentences.
It talks about how compression works in the context of LLMs.
The goal is to reduce token usage while preserving meaning.
There are several layers involved in the pipeline.
Each layer contributes to the overall compression ratio.`

	output, stats := tok.Compress(input, tok.Minimal)

	if output == "" {
		t.Fatal("full pipeline (minimal) returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero")
	}
	if stats.FinalTokens == 0 {
		t.Fatal("FinalTokens should be non-zero")
	}
	// Output must be shorter than or equal to the input
	if len(output) > len(input) {
		t.Errorf("output length (%d) exceeds input length (%d)", len(output), len(input))
	}
}

func TestIntegration_FullPipeline_Aggressive(t *testing.T) {
	input := `This is a fairly long piece of text that contains multiple sentences.
It talks about how compression works in the context of LLMs.
The goal is to reduce token usage while preserving meaning.
There are several layers involved in the pipeline.
Each layer contributes to the overall compression ratio.
Additional filler content follows to give the compressor something to work with.
The quick brown fox jumps over the lazy dog.
Pack my box with five dozen liquor jugs.
How vexingly quick daft zebras jump.`

	output, stats := tok.Compress(input, tok.Aggressive)

	if output == "" {
		t.Fatal("full pipeline (aggressive) returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero")
	}
	// Aggressive mode should achieve some reduction on a long enough input
	if stats.ReductionPercent < 0 {
		t.Errorf("ReductionPercent should not be negative, got %.2f", stats.ReductionPercent)
	}
}

func TestIntegration_FullPipeline_WithQueryIntent(t *testing.T) {
	input := `[INFO] Starting application server on port 8080
[INFO] Connected to database successfully
[DEBUG] Loading configuration from /etc/app/config.yaml
[WARN] Cache miss for key user_session_12345
[INFO] Request received: GET /api/users/42
[ERROR] Connection refused to upstream service
[WARN] Retrying connection attempt 1 of 3
[INFO] Connection restored after retry
[DEBUG] Response sent: 200 OK in 45ms
[INFO] Health check passed`

	output, stats := tok.Compress(input, tok.WithQuery("find errors"))

	if output == "" {
		t.Fatal("query-aware compression returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero")
	}
	// The error line should still be present (query intent targets errors)
	if !strings.Contains(output, "ERROR") && !strings.Contains(output, "error") {
		t.Log("warning: query intent 'find errors' did not preserve the ERROR line")
	}
}

func TestIntegration_FullPipeline_AllTiers(t *testing.T) {
	input := strings.Repeat("line of content with enough words to trigger compression\n", 50)

	tiers := []struct {
		name string
		tier tok.Tier
	}{
		{"surface", tok.TierSurface},
		{"code", tok.TierCode},
		{"log", tok.TierLog},
		{"adaptive", tok.TierAdaptive},
	}

	for _, tt := range tiers {
		t.Run(tt.name, func(t *testing.T) {
			output, stats := tok.Compress(input, tok.WithTier(tt.tier))
			if output == "" {
				t.Errorf("tier %s returned empty output", tt.name)
			}
			if stats.OriginalTokens == 0 {
				t.Errorf("tier %s: OriginalTokens is zero", tt.name)
			}
		})
	}
}

// ---------------------------------------------------------------------------
// 2. Reversibility - Test that compression is deterministic and that
//    CompactionSchema round-trips correctly through JSON serialization.
// ---------------------------------------------------------------------------

func TestIntegration_Reversibility_DeterministicCompression(t *testing.T) {
	input := "Deterministic compression test: same input should produce same output every time."

	// Run compression twice with the same config
	out1, stats1 := tok.Compress(input, tok.Minimal)
	out2, stats2 := tok.Compress(input, tok.Minimal)

	if out1 != out2 {
		t.Errorf("compression is not deterministic:\n  run1: %q\n  run2: %q", out1, out2)
	}
	if stats1.OriginalTokens != stats2.OriginalTokens {
		t.Errorf("OriginalTokens differ: %d vs %d", stats1.OriginalTokens, stats2.OriginalTokens)
	}
	if stats1.FinalTokens != stats2.FinalTokens {
		t.Errorf("FinalTokens differ: %d vs %d", stats1.FinalTokens, stats2.FinalTokens)
	}
}

func TestIntegration_Reversibility_DeterministicAcrossModes(t *testing.T) {
	input := "Test content that will be compressed in both minimal and aggressive modes."

	outMin, statsMin := tok.Compress(input, tok.Minimal)
	outAgg, statsAgg := tok.Compress(input, tok.Aggressive)

	// Both should produce valid output
	if outMin == "" || outAgg == "" {
		t.Fatal("both modes should produce non-empty output")
	}
	// Different modes may produce different results
	if statsMin.OriginalTokens != statsAgg.OriginalTokens {
		t.Errorf("OriginalTokens should be identical across modes: %d vs %d",
			statsMin.OriginalTokens, statsAgg.OriginalTokens)
	}
}

func TestIntegration_Reversibility_CompactionSchemaRoundTrip(t *testing.T) {
	original := &tok.CompactionSchema{
		TaskOverview:         "Build test suite",
		CurrentState:         "Writing integration tests",
		ImportantDiscoveries: []string{"Pipeline has 20 layers", "Supports multiple tiers"},
		NextSteps:            []string{"Run all tests", "Fix any failures"},
		ContextToPreserve:    []string{"File: integration_test.go", "Module: github.com/GrayCodeAI/tok"},
	}

	// Serialize to prompt and re-parse (simulating LLM round-trip)
	prompt := original.ToPrompt()
	if prompt == "" {
		t.Fatal("ToPrompt returned empty string")
	}

	// Simulate an LLM JSON response based on the schema
	jsonResponse := `{
		"task_overview": "Build test suite",
		"current_state": "Writing integration tests",
		"important_discoveries": ["Pipeline has 20 layers", "Supports multiple tiers"],
		"next_steps": ["Run all tests", "Fix any failures"],
		"context_to_preserve": ["File: integration_test.go", "Module: github.com/GrayCodeAI/tok"]
	}`

	parsed, err := tok.ParseCompactionResponse(jsonResponse)
	if err != nil {
		t.Fatalf("ParseCompactionResponse failed: %v", err)
	}

	// Verify round-trip fidelity
	if parsed.TaskOverview != original.TaskOverview {
		t.Errorf("TaskOverview mismatch: %q vs %q", parsed.TaskOverview, original.TaskOverview)
	}
	if parsed.CurrentState != original.CurrentState {
		t.Errorf("CurrentState mismatch: %q vs %q", parsed.CurrentState, original.CurrentState)
	}
	if len(parsed.ImportantDiscoveries) != len(original.ImportantDiscoveries) {
		t.Errorf("ImportantDiscoveries length mismatch: %d vs %d",
			len(parsed.ImportantDiscoveries), len(original.ImportantDiscoveries))
	}
	if len(parsed.NextSteps) != len(original.NextSteps) {
		t.Errorf("NextSteps length mismatch: %d vs %d",
			len(parsed.NextSteps), len(original.NextSteps))
	}
	if len(parsed.ContextToPreserve) != len(original.ContextToPreserve) {
		t.Errorf("ContextToPreserve length mismatch: %d vs %d",
			len(parsed.ContextToPreserve), len(original.ContextToPreserve))
	}
}

func TestIntegration_Reversibility_SecretDetectionAndRedaction(t *testing.T) {
	detector := tok.DefaultSecretDetector()

	textWithSecrets := "My AWS key is AKIAIOSFODNN7EXAMPLE and my GitHub token is ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij"

	matches := detector.DetectSecrets(textWithSecrets)
	if len(matches) == 0 {
		t.Fatal("should detect at least one secret")
	}

	redacted := detector.RedactSecrets(textWithSecrets)
	if strings.Contains(redacted, "AKIAIOSFODNN7EXAMPLE") {
		t.Error("redacted text should not contain the AWS key")
	}
	if strings.Contains(redacted, "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij") {
		t.Error("redacted text should not contain the GitHub token")
	}
	if !strings.Contains(redacted, "REDACTED") {
		t.Error("redacted text should contain REDACTED markers")
	}
}

// ---------------------------------------------------------------------------
// 3. Token Estimation - Test that token counts are accurate after compression
// ---------------------------------------------------------------------------

func TestIntegration_TokenEstimation_Accurate(t *testing.T) {
	// Short text: token count should be roughly len/4 (BPE heuristic)
	short := "Hello world"
	tokens := tok.EstimateTokens(short)
	if tokens <= 0 {
		t.Error("token estimate for short text should be positive")
	}
	if tokens > 10 {
		t.Errorf("token estimate for %q seems too high: %d", short, tokens)
	}
}

func TestIntegration_TokenEstimation_ScalesWithLength(t *testing.T) {
	small := tok.EstimateTokens("hello")
	large := tok.EstimateTokens(strings.Repeat("hello world test sentence. ", 100))

	if large <= small {
		t.Errorf("large text tokens (%d) should exceed small text tokens (%d)", large, small)
	}
}

func TestIntegration_TokenEstimation_AfterCompression(t *testing.T) {
	input := strings.Repeat("The quick brown fox jumps over the lazy dog. ", 50)
	output, stats := tok.Compress(input, tok.Aggressive)

	// The stats should reflect accurate token counting
	if stats.OriginalTokens <= 0 {
		t.Fatal("OriginalTokens should be positive")
	}
	if stats.FinalTokens <= 0 {
		t.Fatal("FinalTokens should be positive")
	}

	// FinalTokens should approximately match the actual token count of the output
	actualFinal := tok.EstimateTokens(output)
	diff := stats.FinalTokens - actualFinal
	if diff < -5 || diff > 5 {
		t.Logf("warning: stats.FinalTokens=%d but EstimateTokens(output)=%d (diff=%d)", stats.FinalTokens, actualFinal, diff)
	}

	// TokensSaved should equal OriginalTokens - FinalTokens
	expectedSaved := stats.OriginalTokens - stats.FinalTokens
	if expectedSaved < 0 {
		expectedSaved = 0
	}
	if stats.TokensSaved != expectedSaved {
		t.Errorf("TokensSaved=%d, but Original-Final=%d", stats.TokensSaved, expectedSaved)
	}
}

func TestIntegration_TokenEstimation_EmptyString(t *testing.T) {
	tokens := tok.EstimateTokens("")
	if tokens != 0 {
		t.Errorf("empty string should have 0 tokens, got %d", tokens)
	}
}

func TestIntegration_TokenEstimation_CodeSnippet(t *testing.T) {
	code := `func main() {
	fmt.Println("hello world")
	x := 42
	if x > 0 {
		return
	}
}`
	tokens := tok.EstimateTokens(code)
	if tokens <= 0 {
		t.Error("code snippet should have positive token count")
	}
	// Rough sanity: a ~100 byte snippet should be 20-60 tokens
	if tokens > 100 {
		t.Errorf("token count for small code snippet seems too high: %d", tokens)
	}
}

// ---------------------------------------------------------------------------
// 4. Language Detection - Test that language-specific compression works
// ---------------------------------------------------------------------------

func TestIntegration_LanguageDetection_Go(t *testing.T) {
	goCode := `package main

import "fmt"

func main() {
	fmt.Println("hello world")
}`

	output, _ := tok.Compress(goCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of Go code returned empty output")
	}
	// Go-specific structures should be preserved
	if !strings.Contains(output, "func") {
		t.Error("Go code compression should preserve 'func' keyword")
	}
}

func TestIntegration_LanguageDetection_Python(t *testing.T) {
	pythonCode := `def hello_world():
    print("Hello, World!")

class MyClass:
    def __init__(self):
        self.value = 42

    def get_value(self):
        return self.value`

	output, _ := tok.Compress(pythonCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of Python code returned empty output")
	}
}

func TestIntegration_LanguageDetection_TypeScript(t *testing.T) {
	tsCode := `interface User {
    name: string;
    age: number;
}

const greet = (user: User): void => {
    console.log("Hello " + user.name);
};

export default greet;`

	output, _ := tok.Compress(tsCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of TypeScript code returned empty output")
	}
}

func TestIntegration_LanguageDetection_Rust(t *testing.T) {
	rustCode := `fn main() {
    let greeting: &str = "Hello, world!";
    println!("{}", greeting);
}

pub fn add(a: i32, b: i32) -> i32 {
    a + b
}`

	output, _ := tok.Compress(rustCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of Rust code returned empty output")
	}
	if !strings.Contains(output, "fn") {
		t.Error("Rust code compression should preserve 'fn' keyword")
	}
}

func TestIntegration_LanguageDetection_SQL(t *testing.T) {
	sqlCode := `SELECT u.name, u.email, COUNT(o.id) as order_count
FROM users u
LEFT JOIN orders o ON u.id = o.user_id
WHERE u.active = true
GROUP BY u.name, u.email
ORDER BY order_count DESC;`

	output, _ := tok.Compress(sqlCode, tok.Code)
	if output == "" {
		t.Fatal("code-tier compression of SQL returned empty output")
	}
}

func TestIntegration_LanguageDetection_LogContent(t *testing.T) {
	logContent := `[2026-05-28 10:00:01] [INFO] Application started
[2026-05-28 10:00:02] [INFO] Connected to database
[2026-05-28 10:00:03] [WARN] Slow query detected (450ms)
[2026-05-28 10:00:04] [INFO] Request processed successfully
[2026-05-28 10:00:05] [ERROR] Connection timeout to service-x
[2026-05-28 10:00:06] [INFO] Retrying connection
[2026-05-28 10:00:07] [INFO] Connection restored
[2026-05-28 10:00:08] [DEBUG] Cache hit ratio: 87.3%
[2026-05-28 10:00:09] [INFO] Health check passed
[2026-05-28 10:00:10] [INFO] Metrics exported`

	output, stats := tok.Compress(logContent, tok.Log)
	if output == "" {
		t.Fatal("log-tier compression returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero for log content")
	}
}

func TestIntegration_LanguageDetection_ConversationContent(t *testing.T) {
	conversation := `User: Can you help me write a function to sort a list?
Assistant: Sure! Here's a quick sort implementation in Python:
def quicksort(arr):
    if len(arr) <= 1:
        return arr
    pivot = arr[len(arr) // 2]
    left = [x for x in arr if x < pivot]
    middle = [x for x in arr if x == pivot]
    right = [x for x in arr if x > pivot]
    return quicksort(left) + middle + quicksort(right)
User: Thanks! Can you also add error handling?
Assistant: Of course, I've added a try-except block around the main logic.`

	output, _ := tok.Compress(conversation, tok.Adaptive)
	if output == "" {
		t.Fatal("adaptive compression of conversation returned empty output")
	}
}

// ---------------------------------------------------------------------------
// 5. Edge Cases - Test empty input, single-word, very large input
// ---------------------------------------------------------------------------

func TestIntegration_EdgeCase_EmptyInput(t *testing.T) {
	output, stats := tok.Compress("")
	if output != "" {
		t.Errorf("empty input should return empty output, got %q", output)
	}
	if stats.OriginalTokens != 0 {
		t.Errorf("empty input should have 0 original tokens, got %d", stats.OriginalTokens)
	}
	if stats.FinalTokens != 0 {
		t.Errorf("empty input should have 0 final tokens, got %d", stats.FinalTokens)
	}
}

func TestIntegration_EdgeCase_SingleWord(t *testing.T) {
	output, stats := tok.Compress("hello")
	if output == "" {
		t.Fatal("single-word input should not return empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("single-word input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_SingleCharacter(t *testing.T) {
	output, stats := tok.Compress("x")
	if output == "" {
		t.Fatal("single-character input should not return empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("single-character input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_OnlyWhitespace(t *testing.T) {
	output, _ := tok.Compress("   \n\n\t  \n   ")
	// Whitespace-only input should produce some output (pipeline doesn't strip everything)
	// The important thing is it doesn't panic
	_ = output
}

func TestIntegration_EdgeCase_OnlyNewlines(t *testing.T) {
	input := strings.Repeat("\n", 100)
	output, stats := tok.Compress(input, tok.Aggressive)
	// Should not panic; output may be empty or short
	if stats.OriginalTokens < 0 {
		t.Error("OriginalTokens should not be negative")
	}
	_ = output
}

func TestIntegration_EdgeCase_VeryLargeInput(t *testing.T) {
	// 100 KB of repetitive text
	input := strings.Repeat("This is a line of text that repeats many times for testing purposes.\n", 5000)

	output, stats := tok.Compress(input, tok.Aggressive)

	if output == "" {
		t.Fatal("very large input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("very large input should have non-zero original tokens")
	}
	// Should achieve significant compression on repetitive input
	if len(output) >= len(input) {
		t.Error("compressor should reduce repetitive large input")
	}
}

func TestIntegration_EdgeCase_UnicodeContent(t *testing.T) {
	input := "Unicode test: éèê üöä 世界 Привет مرحبا 😀😁😂"
	output, stats := tok.Compress(input)
	if output == "" {
		t.Fatal("unicode input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("unicode input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_JSONContent(t *testing.T) {
	input := `{
  "name": "test-application",
  "version": "1.0.0",
  "description": "A test application for compression pipeline integration tests",
  "main": "index.js",
  "scripts": {
    "start": "node index.js",
    "test": "jest --coverage",
    "build": "webpack --mode production",
    "lint": "eslint src/**/*.js"
  },
  "dependencies": {
    "express": "^4.18.0",
    "lodash": "^4.17.21",
    "moment": "^2.29.4"
  },
  "devDependencies": {
    "jest": "^29.0.0",
    "eslint": "^8.0.0",
    "webpack": "^5.0.0"
  }
}`
	output, stats := tok.Compress(input)
	if output == "" {
		t.Fatal("JSON input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("JSON input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_DiffContent(t *testing.T) {
	input := `diff --git a/src/main.go b/src/main.go
index 1234567..abcdefg 100644
--- a/src/main.go
+++ b/src/main.go
@@ -10,6 +10,8 @@ import (
 	"fmt"
 	"os"
+	"log"
+	"net/http"
 )

-func main() {
+func main() {
+	http.HandleFunc("/", handler)
 	fmt.Println("hello")
+	log.Println("server starting")
 }`
	output, stats := tok.Compress(input)
	if output == "" {
		t.Fatal("diff input returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Error("diff input should have non-zero original tokens")
	}
}

func TestIntegration_EdgeCase_RepetitiveContent(t *testing.T) {
	input := strings.Repeat("the same line over and over again\n", 200)
	output, stats := tok.Compress(input, tok.Aggressive)

	if output == "" {
		t.Fatal("repetitive input returned empty output")
	}
	// Repetitive content should compress well
	if stats.ReductionPercent < 0 {
		t.Errorf("ReductionPercent should not be negative for repetitive input: %.2f", stats.ReductionPercent)
	}
}

// ---------------------------------------------------------------------------
// 6. Performance - Test that compression completes within reasonable time
// ---------------------------------------------------------------------------

func TestIntegration_Performance_SmallInput(t *testing.T) {
	input := "Short text for performance test."
	deadline := 500 * time.Millisecond

	start := time.Now()
	tok.Compress(input)
	elapsed := time.Since(start)

	if elapsed > deadline {
		t.Errorf("small input compression took %v, expected under %v", elapsed, deadline)
	}
}

func TestIntegration_Performance_MediumInput(t *testing.T) {
	input := strings.Repeat("This is a medium-length sentence for performance testing. ", 100)
	deadline := 2 * time.Second

	start := time.Now()
	tok.Compress(input, tok.Aggressive)
	elapsed := time.Since(start)

	if elapsed > deadline {
		t.Errorf("medium input compression took %v, expected under %v", elapsed, deadline)
	}
}

func TestIntegration_Performance_LargeInput(t *testing.T) {
	// ~100 KB of content
	input := strings.Repeat("This is a line of text that will be compressed by the full pipeline. It has enough content to exercise multiple layers. ", 2000)
	deadline := 10 * time.Second

	start := time.Now()
	tok.Compress(input, tok.Aggressive)
	elapsed := time.Since(start)

	if elapsed > deadline {
		t.Errorf("large input compression took %v, expected under %v", elapsed, deadline)
	}
}

func TestIntegration_Performance_CodeInput(t *testing.T) {
	var sb strings.Builder
	for i := 0; i < 100; i++ {
		sb.WriteString("func process")
		sb.WriteString(strings.Repeat("x", 10))
		sb.WriteString("(input string) string {\n")
		sb.WriteString("\tresult := strings.TrimSpace(input)\n")
		sb.WriteString("\tresult = strings.ToLower(result)\n")
		sb.WriteString("\treturn result\n")
		sb.WriteString("}\n\n")
	}
	input := sb.String()
	deadline := 5 * time.Second

	start := time.Now()
	tok.Compress(input, tok.Code)
	elapsed := time.Since(start)

	if elapsed > deadline {
		t.Errorf("code input compression took %v, expected under %v", elapsed, deadline)
	}
}

func TestIntegration_Performance_RepeatedCalls(t *testing.T) {
	input := "This is a sentence for testing repeated compression calls with the same input."
	deadline := 5 * time.Second

	start := time.Now()
	for i := 0; i < 100; i++ {
		tok.Compress(input)
	}
	elapsed := time.Since(start)

	if elapsed > deadline {
		t.Errorf("100 repeated compressions took %v, expected under %v", elapsed, deadline)
	}
}

func TestIntegration_Performance_CompressorReuse(t *testing.T) {
	c := tok.NewCompressor(tok.Adaptive)
	inputs := []string{
		"First input to the reusable compressor.",
		"Second input with different content for comparison.",
		"Third input that is a bit longer to test how the pipeline handles varied sizes across calls.",
		strings.Repeat("Fourth input is quite large. ", 50),
	}

	deadline := 5 * time.Second
	start := time.Now()
	for _, input := range inputs {
		output, stats := c.Compress(input)
		if output == "" {
			t.Fatal("reusable compressor returned empty output")
		}
		if stats.OriginalTokens == 0 {
			t.Fatal("reusable compressor returned zero OriginalTokens")
		}
	}
	elapsed := time.Since(start)

	if elapsed > deadline {
		t.Errorf("4 compressions with reused compressor took %v, expected under %v", elapsed, deadline)
	}
}

// ---------------------------------------------------------------------------
// 7. Configuration - Test that pipeline configuration affects output correctly
// ---------------------------------------------------------------------------

func TestIntegration_Configuration_BudgetEnforcement(t *testing.T) {
	input := strings.Repeat("word ", 500) // ~500 words

	_, tightStats := tok.Compress(input, tok.WithBudget(50))
	_, looseStats := tok.Compress(input, tok.WithBudget(500))

	if tightStats.FinalTokens > 70 {
		t.Errorf("tight budget (50) produced %d tokens, expected near 50", tightStats.FinalTokens)
	}
	// Loose budget should preserve more tokens
	if looseStats.FinalTokens < tightStats.FinalTokens {
		t.Errorf("loose budget (%d) produced fewer tokens (%d) than tight budget (%d)",
			500, looseStats.FinalTokens, tightStats.FinalTokens)
	}
}

func TestIntegration_Configuration_MinimalVsAggressive(t *testing.T) {
	input := strings.Repeat("This is test content for comparing compression modes. ", 100)

	_, minimalStats := tok.Compress(input, tok.Minimal)
	_, aggressiveStats := tok.Compress(input, tok.Aggressive)

	if minimalStats.OriginalTokens == 0 || aggressiveStats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero for both modes")
	}

	// Aggressive mode should produce fewer or equal tokens compared to minimal
	if aggressiveStats.FinalTokens > minimalStats.FinalTokens+10 { // allow small margin
		t.Errorf("aggressive mode (%d tokens) should not produce more than minimal mode (%d tokens)",
			aggressiveStats.FinalTokens, minimalStats.FinalTokens)
	}
}

func TestIntegration_Configuration_CodeVsLogTier(t *testing.T) {
	codeInput := `func main() {
	fmt.Println("hello")
	x := 42
	return x
}

func helper(s string) string {
	return strings.TrimSpace(s)
}`

	_, codeStats := tok.Compress(codeInput, tok.Code)
	_, logStats := tok.Compress(codeInput, tok.Log)

	// Both should produce non-empty output
	if codeStats.OriginalTokens == 0 || logStats.OriginalTokens == 0 {
		t.Fatal("both tiers should report non-zero OriginalTokens")
	}
	// Stats may differ based on tier-specific layers
	if codeStats.FinalTokens == 0 {
		t.Error("code tier should produce non-zero FinalTokens")
	}
	if logStats.FinalTokens == 0 {
		t.Error("log tier should produce non-zero FinalTokens")
	}
}

func TestIntegration_Configuration_WithAndWithoutQuery(t *testing.T) {
	input := `[INFO] Application started
[ERROR] Failed to connect to database
[WARN] Retrying connection
[INFO] Connected successfully
[ERROR] Timeout on request to /api/data
[INFO] Retrying request
[INFO] Request succeeded`

	_, noQueryStats := tok.Compress(input)
	_, withQueryStats := tok.Compress(input, tok.WithQuery("database errors"))

	if noQueryStats.OriginalTokens == 0 || withQueryStats.OriginalTokens == 0 {
		t.Fatal("OriginalTokens should be non-zero")
	}
	// Query-aware compression may produce different results (more or fewer tokens
	// depending on relevance scoring); both should be valid
	if noQueryStats.FinalTokens == 0 {
		t.Error("no-query compression should produce non-zero FinalTokens")
	}
	if withQueryStats.FinalTokens == 0 {
		t.Error("query-aware compression should produce non-zero FinalTokens")
	}
}

func TestIntegration_Configuration_AdaptiveTier_AutoDetectsContentType(t *testing.T) {
	logInput := strings.Repeat("[INFO] 2026-05-28T10:00:00Z level=info msg=\"request processed\" status=200\n", 100)
	_, logStats := tok.Compress(logInput, tok.Adaptive)
	if logStats.OriginalTokens == 0 {
		t.Fatal("adaptive tier should report non-zero OriginalTokens for log content")
	}

	codeInput := strings.Repeat("func processItem(id int) error {\n\treturn nil\n}\n\n", 50)
	_, codeStats := tok.Compress(codeInput, tok.Adaptive)
	if codeStats.OriginalTokens == 0 {
		t.Fatal("adaptive tier should report non-zero OriginalTokens for code content")
	}
}

func TestIntegration_Configuration_SurfaceTier_FastPath(t *testing.T) {
	input := strings.Repeat("content line for surface tier testing\n", 100)

	start := time.Now()
	output, stats := tok.Compress(input, tok.Surface)
	elapsed := time.Since(start)

	if output == "" {
		t.Fatal("surface tier returned empty output")
	}
	if stats.OriginalTokens == 0 {
		t.Fatal("surface tier should report non-zero OriginalTokens")
	}
	// Surface tier (4 layers) should be fast
	if elapsed > 2*time.Second {
		t.Errorf("surface tier took %v, expected under 2s for medium input", elapsed)
	}
}

func TestIntegration_Configuration_StatsLayerBreakdown(t *testing.T) {
	input := strings.Repeat("test content for layer stats verification\n", 100)
	_, stats := tok.Compress(input, tok.Minimal)

	if stats.Layers == nil {
		t.Fatal("stats.Layers should not be nil")
	}
	if len(stats.Layers) == 0 {
		t.Error("stats.Layers should contain at least one layer stat")
	}
	// Verify that reported layers have reasonable values
	for name, ls := range stats.Layers {
		if ls.TokensSaved < 0 {
			t.Errorf("layer %q has negative TokensSaved: %d", name, ls.TokensSaved)
		}
		if ls.DurationMs < 0 {
			t.Errorf("layer %q has negative DurationMs: %d", name, ls.DurationMs)
		}
	}
}

// ---------------------------------------------------------------------------
// Additional integration tests: Compressor reuse, concurrent safety
// ---------------------------------------------------------------------------

func TestIntegration_Compressor_ReuseAcrossInputs(t *testing.T) {
	c := tok.NewCompressor(tok.Minimal)

	inputs := []string{
		"First unique input for reuse testing.",
		"Second completely different content here.",
		"",
		"Fourth input after empty.",
	}

	for i, input := range inputs {
		output, stats := c.Compress(input)
		if input == "" {
			if output != "" {
				t.Errorf("call %d: empty input should produce empty output", i)
			}
			continue
		}
		if output == "" {
			t.Errorf("call %d: non-empty input produced empty output", i)
		}
		if stats.OriginalTokens == 0 {
			t.Errorf("call %d: OriginalTokens is zero for non-empty input", i)
		}
	}
}

func TestIntegration_ConcurrentCompression(t *testing.T) {
	input := strings.Repeat("concurrent safety test content with enough words\n", 50)

	done := make(chan bool, 20)
	for i := 0; i < 20; i++ {
		go func() {
			output, stats := tok.Compress(input)
			if output == "" {
				t.Error("concurrent compression returned empty output")
			}
			if stats.OriginalTokens == 0 {
				t.Error("concurrent compression returned zero OriginalTokens")
			}
			done <- true
		}()
	}

	for i := 0; i < 20; i++ {
		select {
		case <-done:
		case <-time.After(10 * time.Second):
			t.Fatal("concurrent compression timed out")
		}
	}
}

func TestIntegration_ConcurrentCompressor(t *testing.T) {
	c := tok.NewCompressor(tok.Adaptive)
	input := strings.Repeat("compressor concurrent test input with enough content\n", 50)

	done := make(chan bool, 10)
	for i := 0; i < 10; i++ {
		go func() {
			output, stats := c.Compress(input)
			if output == "" {
				t.Error("concurrent compressor returned empty output")
			}
			if stats.OriginalTokens == 0 {
				t.Error("concurrent compressor returned zero OriginalTokens")
			}
			done <- true
		}()
	}

	for i := 0; i < 10; i++ {
		select {
		case <-done:
		case <-time.After(10 * time.Second):
			t.Fatal("concurrent compressor timed out")
		}
	}
}

// ---------------------------------------------------------------------------
// Additional integration tests: CompactionSchema and BuildCompactionPrompt
// ---------------------------------------------------------------------------

func TestIntegration_CompactionSchema_ToPrompt(t *testing.T) {
	schema := &tok.CompactionSchema{
		TaskOverview:         "Implement compression pipeline tests",
		CurrentState:         "Writing integration tests for the tok package",
		ImportantDiscoveries: []string{"Pipeline has 20 layers", "Supports multiple tiers"},
		NextSteps:            []string{"Run tests", "Fix failures"},
		ContextToPreserve:    []string{"File: integration_test.go", "Module: github.com/GrayCodeAI/tok"},
	}

	prompt := schema.ToPrompt()
	if prompt == "" {
		t.Fatal("ToPrompt returned empty string")
	}
	if !strings.Contains(prompt, "Task Overview") {
		t.Error("prompt should contain 'Task Overview' section")
	}
	if !strings.Contains(prompt, "Important Discoveries") {
		t.Error("prompt should contain 'Important Discoveries' section")
	}
	if !strings.Contains(prompt, "compression pipeline") {
		t.Error("prompt should contain the task overview text")
	}
}

func TestIntegration_CompactionSchema_ParseResponse(t *testing.T) {
	jsonResponse := `{
		"task_overview": "Building test suite",
		"current_state": "Almost done",
		"important_discoveries": ["Finding 1", "Finding 2"],
		"next_steps": ["Run tests"],
		"context_to_preserve": ["key detail"]
	}`

	schema, err := tok.ParseCompactionResponse(jsonResponse)
	if err != nil {
		t.Fatalf("ParseCompactionResponse failed: %v", err)
	}
	if schema.TaskOverview != "Building test suite" {
		t.Errorf("TaskOverview = %q, want %q", schema.TaskOverview, "Building test suite")
	}
	if len(schema.ImportantDiscoveries) != 2 {
		t.Errorf("ImportantDiscoveries count = %d, want 2", len(schema.ImportantDiscoveries))
	}
}

func TestIntegration_CompactionSchema_ParseMarkdownFencedResponse(t *testing.T) {
	fencedResponse := "```json\n{\n\t\"task_overview\": \"Test\",\n\t\"current_state\": \"Done\"\n}\n```"

	schema, err := tok.ParseCompactionResponse(fencedResponse)
	if err != nil {
		t.Fatalf("ParseCompactionResponse with markdown fences failed: %v", err)
	}
	if schema.TaskOverview != "Test" {
		t.Errorf("TaskOverview = %q, want %q", schema.TaskOverview, "Test")
	}
}

func TestIntegration_BuildCompactionPrompt(t *testing.T) {
	prompt := tok.BuildCompactionPrompt("some context to compress", 0)
	if prompt == "" {
		t.Fatal("BuildCompactionPrompt returned empty string")
	}
	if !strings.Contains(prompt, "some context to compress") {
		t.Error("prompt should contain the input context")
	}
	if !strings.Contains(prompt, "task_overview") {
		t.Error("prompt should contain schema field names")
	}
}

func TestIntegration_BuildCompactionPrompt_Truncation(t *testing.T) {
	longContext := strings.Repeat("x", 10000)
	prompt := tok.BuildCompactionPrompt(longContext, 1000)
	// The context portion should be truncated to maxChars
	if len(prompt) > 2000 { // system prompt + truncated context
		t.Errorf("prompt length %d exceeds expected maximum", len(prompt))
	}
}

// ---------------------------------------------------------------------------
// Additional integration tests: Token estimation precision
// ---------------------------------------------------------------------------

func TestIntegration_TokenEstimation_PreciseVsFast(t *testing.T) {
	// EstimateTokens uses BPE (precise); we verify it returns positive values
	// for a variety of content types
	contents := []string{
		"Hello, world!",
		"func main() { fmt.Println(\"test\") }",
		"The quick brown fox jumps over the lazy dog.",
		strings.Repeat("a", 1000),
		"JSON: {\"key\": \"value\", \"array\": [1, 2, 3]}",
	}

	for _, c := range contents {
		tokens := tok.EstimateTokens(c)
		if tokens <= 0 {
			t.Errorf("EstimateTokens(%q) = %d, want > 0", c[:min(30, len(c))], tokens)
		}
	}
}

func TestIntegration_TokenEstimation_Monotonicity(t *testing.T) {
	// Longer text should generally have more tokens
	short := "Hello"
	medium := "Hello, this is a medium-length sentence for testing."
	long := "Hello, this is a much longer sentence that goes on and on with many more words and details about testing token estimation monotonicity properties."

	shortTok := tok.EstimateTokens(short)
	medTok := tok.EstimateTokens(medium)
	longTok := tok.EstimateTokens(long)

	if medTok < shortTok {
		t.Errorf("medium tokens (%d) should be >= short tokens (%d)", medTok, shortTok)
	}
	if longTok < medTok {
		t.Errorf("long tokens (%d) should be >= medium tokens (%d)", longTok, medTok)
	}
}