Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"db:studio": "pnpm --filter db db:studio"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.92.0",
"@payloadcms/admin-bar": "3.64.0",
"@payloadcms/db-postgres": "3.64.0",
"@payloadcms/live-preview-react": "3.64.0",
Expand Down
32 changes: 32 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 10 additions & 21 deletions src/lib/replyTriage.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import Anthropic from "@anthropic-ai/sdk";
import type { ReplyIntent } from "@coldflow/db";

export interface TriageInput {
Expand All @@ -17,7 +18,6 @@ export interface TriageResult {
}

const CLAUDE_MODEL = process.env.ANTHROPIC_MODEL ?? "claude-haiku-4-5-20251001";
const ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages";

const VALID_INTENTS: ReadonlySet<ReplyIntent> = new Set([
"interested",
Expand Down Expand Up @@ -229,31 +229,20 @@ export const triageReplyLLM = async (
].join("\n");

try {
const res = await fetch(ANTHROPIC_API_URL, {
method: "POST",
signal: opts?.signal,
headers: {
"Content-Type": "application/json",
"x-api-key": apiKey,
"anthropic-version": "2023-06-01",
},
body: JSON.stringify({
const client = new Anthropic({ apiKey });
const message = await client.messages.create(
{
model: CLAUDE_MODEL,
max_tokens: 600,
system: TRIAGE_PROMPT,
messages: [{ role: "user", content: userPrompt }],
}),
});

if (!res.ok) {
console.error("Anthropic triage non-200:", res.status, await res.text());
return null;
}
},
{ signal: opts?.signal }
);

const json = (await res.json()) as {
content?: Array<{ type: string; text?: string }>;
};
const text = json.content?.find((c) => c.type === "text")?.text?.trim();
const text = message.content
.find((block): block is Anthropic.TextBlock => block.type === "text")
?.text.trim();
if (!text) return null;

const parsed = parseLLMJson(text);
Expand Down
123 changes: 43 additions & 80 deletions tests/int/replyTriage.int.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,19 @@ import {
triageReplyLLM,
} from '@/lib/replyTriage'

const { messagesCreate } = vi.hoisted(() => ({ messagesCreate: vi.fn() }))

vi.mock('@anthropic-ai/sdk', () => {
const Anthropic = vi.fn(() => ({
messages: { create: messagesCreate },
}))
return { default: Anthropic }
})

const textResponse = (text: string) => ({
content: [{ type: 'text', text }],
})

describe('triageReplyHeuristic', () => {
it('classifies the labeled cases.json set with >= 80% accuracy', () => {
let correct = 0
Expand Down Expand Up @@ -48,32 +61,23 @@ describe('triageReplyHeuristic', () => {
})

describe('triageReplyLLM', () => {
const originalFetch = globalThis.fetch
beforeEach(() => {
delete process.env.ANTHROPIC_API_KEY
})
afterEach(() => {
globalThis.fetch = originalFetch
messagesCreate.mockReset()
})

it('returns null when no API key is configured', async () => {
const result = await triageReplyLLM({ replyBody: 'price?' })
expect(result).toBeNull()
expect(messagesCreate).not.toHaveBeenCalled()
})

it('parses a valid LLM response', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
content: [
{
type: 'text',
text: '{"intent":"interested","confidence":0.92,"suggested_followup":"Hi Pat, happy to share details."}',
},
],
}),
})
globalThis.fetch = fetchMock as unknown as typeof fetch
messagesCreate.mockResolvedValue(
textResponse(
'{"intent":"interested","confidence":0.92,"suggested_followup":"Hi Pat, happy to share details."}'
)
)

const result = await triageReplyLLM(
{ replyBody: 'price?' },
Expand All @@ -88,18 +92,11 @@ describe('triageReplyLLM', () => {
})

it('tolerates ```json fences in the LLM response', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
content: [
{
type: 'text',
text: '```json\n{"intent":"objection","confidence":0.7,"suggested_followup":"x"}\n```',
},
],
}),
})
globalThis.fetch = fetchMock as unknown as typeof fetch
messagesCreate.mockResolvedValue(
textResponse(
'```json\n{"intent":"objection","confidence":0.7,"suggested_followup":"x"}\n```'
)
)
const result = await triageReplyLLM(
{ replyBody: 'no thanks' },
{ apiKey: 'sk-test' }
Expand All @@ -108,25 +105,16 @@ describe('triageReplyLLM', () => {
})

it('returns null when the LLM returns garbage that cannot be parsed', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ content: [{ type: 'text', text: 'sorry, I cannot help' }] }),
})
globalThis.fetch = fetchMock as unknown as typeof fetch
messagesCreate.mockResolvedValue(textResponse('sorry, I cannot help'))
const result = await triageReplyLLM(
{ replyBody: '...' },
{ apiKey: 'sk-test' }
)
expect(result).toBeNull()
})

it('returns null when the API returns non-200', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: false,
status: 500,
text: async () => 'server error',
})
globalThis.fetch = fetchMock as unknown as typeof fetch
it('returns null when the SDK throws', async () => {
messagesCreate.mockRejectedValue(new Error('500 server error'))
const result = await triageReplyLLM(
{ replyBody: '...' },
{ apiKey: 'sk-test' }
Expand All @@ -135,18 +123,9 @@ describe('triageReplyLLM', () => {
})

it('rejects unknown intent values', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
content: [
{
type: 'text',
text: '{"intent":"happy","confidence":0.9,"suggested_followup":"x"}',
},
],
}),
})
globalThis.fetch = fetchMock as unknown as typeof fetch
messagesCreate.mockResolvedValue(
textResponse('{"intent":"happy","confidence":0.9,"suggested_followup":"x"}')
)
const result = await triageReplyLLM(
{ replyBody: '...' },
{ apiKey: 'sk-test' }
Expand All @@ -155,18 +134,9 @@ describe('triageReplyLLM', () => {
})

it('clamps confidence into [0, 1]', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
content: [
{
type: 'text',
text: '{"intent":"interested","confidence":1.5,"suggested_followup":"x"}',
},
],
}),
})
globalThis.fetch = fetchMock as unknown as typeof fetch
messagesCreate.mockResolvedValue(
textResponse('{"intent":"interested","confidence":1.5,"suggested_followup":"x"}')
)
const result = await triageReplyLLM(
{ replyBody: 'p' },
{ apiKey: 'sk-test' }
Expand All @@ -176,9 +146,10 @@ describe('triageReplyLLM', () => {
})

describe('triageReply (end-to-end)', () => {
const originalFetch = globalThis.fetch
beforeEach(() => {
messagesCreate.mockReset()
})
afterEach(() => {
globalThis.fetch = originalFetch
delete process.env.ANTHROPIC_API_KEY
})

Expand All @@ -191,27 +162,19 @@ describe('triageReply (end-to-end)', () => {

it('falls back to the heuristic when the LLM call fails', async () => {
process.env.ANTHROPIC_API_KEY = 'sk-test'
globalThis.fetch = vi
.fn()
.mockRejectedValue(new Error('network down')) as unknown as typeof fetch
messagesCreate.mockRejectedValue(new Error('network down'))
const result = await triageReply({ replyBody: 'send pricing please' })
expect(result.source).toBe('heuristic')
expect(result.intent).toBe('interested')
})

it('uses LLM result when available', async () => {
process.env.ANTHROPIC_API_KEY = 'sk-test'
globalThis.fetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
content: [
{
type: 'text',
text: '{"intent":"not_now","confidence":0.66,"suggested_followup":"talk later"}',
},
],
}),
}) as unknown as typeof fetch
messagesCreate.mockResolvedValue(
textResponse(
'{"intent":"not_now","confidence":0.66,"suggested_followup":"talk later"}'
)
)
const result = await triageReply({ replyBody: 'busy this quarter' })
expect(result.source).toBe('llm')
expect(result.intent).toBe('not_now')
Expand Down