Add vendor assessment agent with structured output

GNUmakefile

@@ -44,6 +44,18 @@ E2E_COVER_DIR ?= $(CURDIR)/coverage/e2e
 DOCKER_IMAGE_NAME=	ghcr.io/getprobo/probo
 DOCKER_TAG_NAME?=	latest
 
+PROBOD_BIN_DEPS= pkg/server/api/connect/v1/schema/schema.go \
+	pkg/server/api/connect/v1/types/types.go \
+	pkg/server/api/console/v1/schema/schema.go \
+	pkg/server/api/console/v1/types/types.go \
+	pkg/server/api/trust/v1/schema/schema.go \
+	pkg/server/api/trust/v1/types/types.go \
+	pkg/server/api/mcp/v1/server/server.go \
+	pkg/server/api/mcp/v1/types/types.go \
+	apps/console/dist/index.html \
+	apps/trust/dist/index.html \
+	@probo/emails
+
 PROBOD_BIN_EXTRA_DEPS=
 PROBOD_BIN=	bin/probod
 PROBOD_SRC=	cmd/probod/main.go
@@ -127,8 +139,8 @@ test-bench: test ## Run benchmark tests
 
 .PHONY: test-e2e
 test-e2e: CGO_ENABLED=1
-test-e2e: bin/probod ## Run console e2e tests
-	PROBO_E2E_BINARY=$(CURDIR)/bin/probod \
+test-e2e: $(PROBOD_BIN) ## Run console e2e tests
+	PROBO_E2E_BINARY=$(CURDIR)/$(PROBOD_BIN) \
 	PROBO_E2E_CONFIG=$(E2E_CONFIG) \
 	GOTESTSUM_FORMAT=testname $(GO_TEST) -count=1 ./e2e/console/...
 
@@ -152,7 +164,7 @@ coverage-combined: coverage-report test-e2e-coverage ## Generate combined covera
 	$(GO) tool cover -html=coverage-combined.out -o=coverage-combined.html
 
 .PHONY: build
-build: bin/probod bin/prb bin/probod-bootstrap
+build: $(PROBOD_BIN) bin/prb bin/probod-bootstrap
 
 .PHONY: sbom-docker
 sbom-docker: docker-build
@@ -191,19 +203,8 @@ scan-license: ## Check dependencies licenses compliance
 docker-build:
 	$(DOCKER_BUILD) --tag $(DOCKER_IMAGE_NAME):$(DOCKER_TAG_NAME) --file Dockerfile .
 
-.PHONY: bin/probod
-bin/probod: pkg/server/api/connect/v1/schema/schema.go \
-	pkg/server/api/connect/v1/types/types.go \
-	pkg/server/api/console/v1/schema/schema.go \
-	pkg/server/api/console/v1/types/types.go \
-	pkg/server/api/trust/v1/schema/schema.go \
-	pkg/server/api/trust/v1/types/types.go \
-	pkg/server/api/mcp/v1/server/server.go \
-	pkg/server/api/mcp/v1/types/types.go \
-	apps/console/dist/index.html \
-	apps/trust/dist/index.html \
-	$(PROBOD_BIN_EXTRA_DEPS) \
-	@probo/emails
+.PHONY: $(PROBOD_BIN)
+$(PROBOD_BIN): $(PROBOD_BIN_DEPS) $(PROBOD_BIN_EXTRA_DEPS)
 	$(GO_BUILD) -o $(PROBOD_BIN) $(PROBOD_SRC)
 
 .PHONY: bin/prb

cfg/dev.yaml

@@ -73,6 +73,11 @@ probod:
       model-name: gpt-4o
       temperature: 0.1
       max-tokens: 4096
+    # vendor-assessor is opt-in: uncomment to enable AI-driven
+    # vendor assessment. Without this block the feature returns
+    # UNAVAILABLE to callers.
+    # vendor-assessor:
+    #   provider: openai
 
   evidence-describer:
     interval: 10

e2e/console/vendor_test.go

@@ -986,6 +986,118 @@ func TestVendor_OmittableWebsiteUrl(t *testing.T) {
 	})
 }
 
+// TestVendor_Assess exercises the assessVendor mutation through authorization
+// and tenant-isolation paths without running the real LLM/browser pipeline.
+// The e2e config deliberately omits `llm.vendor-assessor.provider`, so an
+// authorized call reaches DisabledVendorAssessor and surfaces a stable
+// UNAVAILABLE error. Happy-path payload shape is covered by unit tests in
+// pkg/probo.
+func TestVendor_Assess(t *testing.T) {
+	t.Parallel()
+
+	const query = `
+		mutation AssessVendor($input: AssessVendorInput!) {
+			assessVendor(input: $input) {
+				vendor {
+					id
+				}
+			}
+		}
+	`
+
+	type resultShape struct {
+		AssessVendor struct {
+			Vendor struct {
+				ID string `json:"id"`
+			} `json:"vendor"`
+		} `json:"assessVendor"`
+	}
+
+	t.Run("owner call surfaces the disabled error", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		vendorID := factory.NewVendor(owner).WithName("Unconfigured assess").Create()
+
+		var result resultShape
+		err := owner.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://vendor.example.com",
+			},
+		}, &result)
+		testutil.RequireErrorCode(t, err, "UNAVAILABLE")
+	})
+
+	t.Run("admin call surfaces the disabled error", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		admin := testutil.NewClientInOrg(t, testutil.RoleAdmin, owner)
+		vendorID := factory.NewVendor(owner).WithName("Admin-assessed vendor").Create()
+
+		var result resultShape
+		err := admin.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://admin.example.com",
+			},
+		}, &result)
+		testutil.RequireErrorCode(t, err, "UNAVAILABLE")
+	})
+
+	t.Run("viewer cannot assess a vendor", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		viewer := testutil.NewClientInOrg(t, testutil.RoleViewer, owner)
+		vendorID := factory.NewVendor(owner).WithName("Viewer attempt").Create()
+
+		var result resultShape
+		err := viewer.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://viewer.example.com",
+			},
+		}, &result)
+		testutil.RequireForbiddenError(t, err)
+	})
+
+	t.Run("cannot assess vendor from another organization", func(t *testing.T) {
+		t.Parallel()
+
+		org1Owner := testutil.NewClient(t, testutil.RoleOwner)
+		org2Owner := testutil.NewClient(t, testutil.RoleOwner)
+		vendorID := factory.NewVendor(org1Owner).WithName("Org1 vendor").Create()
+
+		var result resultShape
+		err := org2Owner.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://cross-tenant.example.com",
+			},
+		}, &result)
+		require.Error(t, err, "vendor assess must not cross tenant boundaries")
+	})
+
+	t.Run("procedure is accepted on the input", func(t *testing.T) {
+		t.Parallel()
+
+		owner := testutil.NewClient(t, testutil.RoleOwner)
+		vendorID := factory.NewVendor(owner).WithName("Procedure test").Create()
+
+		var result resultShape
+		err := owner.Execute(query, map[string]any{
+			"input": map[string]any{
+				"id":         vendorID,
+				"websiteUrl": "https://procedure.example.com",
+				"procedure":  "Focus on SOC 2 controls and data residency",
+			},
+		}, &result)
+		testutil.RequireErrorCode(t, err, "UNAVAILABLE")
+	})
+}
+
 func TestVendor_TenantIsolation(t *testing.T) {
 	t.Parallel()
 

pkg/agent/agent.go

@@ -23,48 +23,53 @@ import (
 	"go.probo.inc/probo/pkg/llm"
 )
 
-const DefaultMaxTurns = 10
+const (
+	DefaultMaxTurns              = 10
+	DefaultMaxEmptyOutputRetries = 2
+)
 
 type (
 	Option func(*Agent)
 
 	Agent struct {
-		name               string
-		handoffDescription string
-		instructions       string
-		instructionsFunc   func(ctx context.Context, a *Agent) string
-		model              string
-		modelSettings      ModelSettings
-		tools              []Tool
-		handoffs           []*Handoff
-		mcpServers         []*MCPServer
-		maxTurns           int
-		maxToolDepth       int
-		client             *llm.Client
-		logger             *log.Logger
-		hooks              []RunHooks
-		agentHooks         AgentHooks
-		inputGuardrails    []InputGuardrail
-		outputGuardrails   []OutputGuardrail
-		session            Session
-		sessionID          string
-		outputType         *OutputType
-		toolUseBehavior    ToolUseBehavior
-		resetToolChoice    bool
-		responseFormat     *llm.ResponseFormat
-		approval           *ApprovalConfig
+		name                  string
+		handoffDescription    string
+		instructions          string
+		instructionsFunc      func(ctx context.Context, a *Agent) string
+		model                 string
+		modelSettings         ModelSettings
+		tools                 []Tool
+		handoffs              []*Handoff
+		mcpServers            []*MCPServer
+		maxTurns              int
+		maxEmptyOutputRetries int
+		maxToolDepth          int
+		client                *llm.Client
+		logger                *log.Logger
+		hooks                 []RunHooks
+		agentHooks            AgentHooks
+		inputGuardrails       []InputGuardrail
+		outputGuardrails      []OutputGuardrail
+		session               Session
+		sessionID             string
+		outputType            *OutputType
+		toolUseBehavior       ToolUseBehavior
+		resetToolChoice       bool
+		responseFormat        *llm.ResponseFormat
+		approval              *ApprovalConfig
 	}
 )
 
 func New(name string, client *llm.Client, opts ...Option) *Agent {
 	a := &Agent{
-		name:            name,
-		client:          client,
-		maxTurns:        DefaultMaxTurns,
-		maxToolDepth:    DefaultMaxToolDepth,
-		toolUseBehavior: RunLLMAgain(),
-		resetToolChoice: true,
-		logger:          log.NewLogger(log.WithOutput(io.Discard)),
+		name:                  name,
+		client:                client,
+		maxTurns:              DefaultMaxTurns,
+		maxEmptyOutputRetries: DefaultMaxEmptyOutputRetries,
+		maxToolDepth:          DefaultMaxToolDepth,
+		toolUseBehavior:       RunLLMAgain(),
+		resetToolChoice:       true,
+		logger:                log.NewLogger(log.WithOutput(io.Discard)),
 	}
 
 	for _, opt := range opts {
@@ -204,6 +209,18 @@ func WithMaxTurns(n int) Option {
 	}
 }
 
+// WithMaxEmptyOutputRetries bounds the number of times the core loop
+// will re-ask the model to produce a structured output after it
+// returned a thinking-only empty response on a synthesis turn.
+func WithMaxEmptyOutputRetries(n int) Option {
+	return func(a *Agent) {
+		if n < 0 {
+			n = 0
+		}
+		a.maxEmptyOutputRetries = n
+	}
+}
+
 func WithMaxToolDepth(n int) Option {
 	return func(a *Agent) {
 		if n < 1 {
@@ -255,6 +272,15 @@ func WithParallelToolCalls(enabled bool) Option {
 	}
 }
 
+func WithThinking(budgetTokens int) Option {
+	return func(a *Agent) {
+		a.modelSettings.Thinking = &llm.ThinkingConfig{
+			Enabled:      true,
+			BudgetTokens: budgetTokens,
+		}
+	}
+}
+
 func WithLogger(l *log.Logger) Option {
 	return func(a *Agent) {
 		a.logger = l