From 376c63e22a0d1247a71a0760c8064200de17d356 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Fri, 6 Mar 2026 17:07:45 +0000 Subject: [PATCH 1/5] add Long descriptions and Example blocks to evaluate commands Co-Authored-By: Claude Opus 4.6 --- TODO.md | 1 + cmd/kosli/evaluate.go | 13 ++++++++--- cmd/kosli/evaluateTrail.go | 44 +++++++++++++++++++++++++++++++----- cmd/kosli/evaluateTrails.go | 45 ++++++++++++++++++++++++++++++++----- 4 files changed, 90 insertions(+), 13 deletions(-) diff --git a/TODO.md b/TODO.md index ff8536f12..ec5887cc7 100644 --- a/TODO.md +++ b/TODO.md @@ -26,3 +26,4 @@ - [x] Slice 16: Extract tree-traversal duplication in transform.go - [x] Slice 17: Align test method naming - [x] Slice 18: Fail evaluation when rehydration errors occur (instead of silently swallowing them) +- [ ] Slice 19: Add Long descriptions and Example blocks to evaluate commands ← active diff --git a/cmd/kosli/evaluate.go b/cmd/kosli/evaluate.go index cb444ad80..c1ba91047 100644 --- a/cmd/kosli/evaluate.go +++ b/cmd/kosli/evaluate.go @@ -6,13 +6,20 @@ import ( "github.com/spf13/cobra" ) -const evaluateDesc = `All Kosli evaluate commands.` +const evaluateShortDesc = `Evaluate Kosli trail data against OPA/Rego policies.` + +const evaluateLongDesc = evaluateShortDesc + ` +Fetch trail data from Kosli and evaluate it against custom policies written +in Rego, the policy language used by Open Policy Agent (OPA). +The policy must use ` + "`package policy`" + ` and define an ` + "`allow`" + ` rule. +An optional ` + "`violations`" + ` rule can provide human-readable denial reasons. +The command exits with code 0 when allowed and code 1 when denied.` func newEvaluateCmd(out io.Writer) *cobra.Command { cmd := &cobra.Command{ Use: "evaluate", - Short: evaluateDesc, - Long: evaluateDesc, + Short: evaluateShortDesc, + Long: evaluateLongDesc, } // Add subcommands diff --git a/cmd/kosli/evaluateTrail.go b/cmd/kosli/evaluateTrail.go index 68692e89d..ba4d5e52c 100644 --- a/cmd/kosli/evaluateTrail.go +++ b/cmd/kosli/evaluateTrail.go @@ -6,7 +6,40 @@ import ( "github.com/spf13/cobra" ) -const evaluateTrailDesc = `Evaluate a trail against a policy.` +const evaluateTrailShortDesc = `Evaluate a trail against a policy.` + +const evaluateTrailLongDesc = evaluateTrailShortDesc + ` +Fetch a single trail from Kosli and evaluate it against a Rego policy using OPA. +The trail data is passed to the policy as ` + "`input.trail`" + `. + +Use ` + "`--attestations`" + ` to enrich the input with detailed attestation data +(e.g. pull request approvers, scan results). Use ` + "`--show-input`" + ` to inspect the +full data structure available to the policy. Use ` + "`--output json`" + ` for structured output.` + +const evaluateTrailExample = ` +# evaluate a trail against a policy: +kosli evaluate trail yourTrailName \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate a trail with attestation enrichment: +kosli evaluate trail yourTrailName \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --attestations pull-request \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate a trail and show the policy input data: +kosli evaluate trail yourTrailName \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --show-input \ + --output json \ + --api-token yourAPIToken \ + --org yourOrgName` type evaluateTrailOptions struct { commonEvaluateOptions @@ -15,10 +48,11 @@ type evaluateTrailOptions struct { func newEvaluateTrailCmd(out io.Writer) *cobra.Command { o := new(evaluateTrailOptions) cmd := &cobra.Command{ - Use: "trail TRAIL-NAME", - Short: evaluateTrailDesc, - Long: evaluateTrailDesc, - Args: cobra.ExactArgs(1), + Use: "trail TRAIL-NAME", + Short: evaluateTrailShortDesc, + Long: evaluateTrailLongDesc, + Example: evaluateTrailExample, + Args: cobra.ExactArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { err := RequireGlobalFlags(global, []string{"Org", "ApiToken"}) if err != nil { diff --git a/cmd/kosli/evaluateTrails.go b/cmd/kosli/evaluateTrails.go index 78c3e3046..ff3ac0a1e 100644 --- a/cmd/kosli/evaluateTrails.go +++ b/cmd/kosli/evaluateTrails.go @@ -6,7 +6,41 @@ import ( "github.com/spf13/cobra" ) -const evaluateTrailsDesc = `Evaluate multiple trails against a policy.` +const evaluateTrailsShortDesc = `Evaluate multiple trails against a policy.` + +const evaluateTrailsLongDesc = evaluateTrailsShortDesc + ` +Fetch multiple trails from Kosli and evaluate them together against a Rego policy using OPA. +The trail data is passed to the policy as ` + "`input.trails`" + ` (an array), unlike +` + "`evaluate trail`" + ` which passes ` + "`input.trail`" + ` (a single object). + +Use ` + "`--attestations`" + ` to enrich the input with detailed attestation data +(e.g. pull request approvers, scan results). Use ` + "`--show-input`" + ` to inspect the +full data structure available to the policy. Use ` + "`--output json`" + ` for structured output.` + +const evaluateTrailsExample = ` +# evaluate multiple trails against a policy: +kosli evaluate trails yourTrailName1 yourTrailName2 \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate trails with attestation enrichment: +kosli evaluate trails yourTrailName1 yourTrailName2 \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --attestations pull-request \ + --api-token yourAPIToken \ + --org yourOrgName + +# evaluate trails with JSON output and show the policy input: +kosli evaluate trails yourTrailName1 yourTrailName2 \ + --policy yourPolicyFile.rego \ + --flow yourFlowName \ + --show-input \ + --output json \ + --api-token yourAPIToken \ + --org yourOrgName` type evaluateTrailsOptions struct { commonEvaluateOptions @@ -15,10 +49,11 @@ type evaluateTrailsOptions struct { func newEvaluateTrailsCmd(out io.Writer) *cobra.Command { o := new(evaluateTrailsOptions) cmd := &cobra.Command{ - Use: "trails TRAIL-NAME [TRAIL-NAME...]", - Short: evaluateTrailsDesc, - Long: evaluateTrailsDesc, - Args: cobra.MinimumNArgs(1), + Use: "trails TRAIL-NAME [TRAIL-NAME...]", + Short: evaluateTrailsShortDesc, + Long: evaluateTrailsLongDesc, + Example: evaluateTrailsExample, + Args: cobra.MinimumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { err := RequireGlobalFlags(global, []string{"Org", "ApiToken"}) if err != nil { From 554c5486fa9f19fc86a50512912bfcd9335c9bae Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Fri, 6 Mar 2026 17:35:54 +0000 Subject: [PATCH 2/5] address PR #671 docs feedback: expand policy contract hint, replace dual-command example with snyk trail example Co-Authored-By: Claude Opus 4.6 --- TODO.md | 2 +- cmd/kosli/evaluate.go | 2 +- .../tutorials/evaluate_trails_with_opa.md | 48 ++++++++++++------- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/TODO.md b/TODO.md index ec5887cc7..c4cdd0cf8 100644 --- a/TODO.md +++ b/TODO.md @@ -26,4 +26,4 @@ - [x] Slice 16: Extract tree-traversal duplication in transform.go - [x] Slice 17: Align test method naming - [x] Slice 18: Fail evaluation when rehydration errors occur (instead of silently swallowing them) -- [ ] Slice 19: Add Long descriptions and Example blocks to evaluate commands ← active +- [x] Slice 19: Add Long descriptions, Example blocks, and docs feedback (policy contract hint, snyk trail example) diff --git a/cmd/kosli/evaluate.go b/cmd/kosli/evaluate.go index c1ba91047..39c013f1d 100644 --- a/cmd/kosli/evaluate.go +++ b/cmd/kosli/evaluate.go @@ -12,7 +12,7 @@ const evaluateLongDesc = evaluateShortDesc + ` Fetch trail data from Kosli and evaluate it against custom policies written in Rego, the policy language used by Open Policy Agent (OPA). The policy must use ` + "`package policy`" + ` and define an ` + "`allow`" + ` rule. -An optional ` + "`violations`" + ` rule can provide human-readable denial reasons. +An optional ` + "`violations`" + ` rule (a set of strings) can provide human-readable denial reasons. The command exits with code 0 when allowed and code 1 when denied.` func newEvaluateCmd(out io.Writer) *cobra.Command { diff --git a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md index dc71a9f45..46a8eb25a 100644 --- a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md +++ b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md @@ -59,7 +59,11 @@ Let's break down what this policy does: - **`allow`** — trails are allowed only when there are no violations. {{}} -The policy contract requires `package policy` and an `allow` rule. The `violations` rule is optional but recommended — it provides human-readable reasons when a trail is denied. +**Policy contract** — these are Kosli-specific conventions, not OPA built-ins: + +- **`package policy`** — required. Kosli queries `data.policy.*` to find your rules. +- **`allow`** — required. Must evaluate to a **boolean**. Kosli exits with code 0 when `true`, code 1 when `false`. +- **`violations`** — optional but recommended. Must be a **set of strings**, where each string is a human-readable reason the policy failed. Kosli displays these when `allow` is `false`. {{}} ## Step 3: Evaluate multiple trails @@ -105,7 +109,9 @@ RESULT: ALLOWED ## Step 4: Evaluate a single trail -To evaluate just one trail, use `kosli evaluate trail` (singular). The data is passed to the policy as `input.trail` instead of `input.trails`, so you need a slightly different policy. Save this as `pr-approved-single.rego`: +The `kosli evaluate trail` (singular) command evaluates facts within a single trail — a different use case from comparing across multiple trails. For example, you might check that a snyk container scan found no high-severity vulnerabilities. + +Save this as `snyk-no-high-vulns.rego`: ```rego package policy @@ -115,9 +121,11 @@ import rego.v1 default allow = false violations contains msg if { - some pr in input.trail.compliance_status.attestations_statuses["pull-request"].pull_requests - count(pr.approvers) == 0 - msg := sprintf("trail '%v': pull-request %v has no approvers", [input.trail.name, pr.url]) + some name, artifact in input.trail.compliance_status.artifacts_statuses + snyk := artifact.attestations_statuses["snyk-container-scan"] + some result in snyk.processed_snyk_results.results + result.high_count > 0 + msg := sprintf("artifact '%v': snyk container scan found %d high severity vulnerabilities", [name, result.high_count]) } allow if { @@ -125,21 +133,27 @@ allow if { } ``` +This policy iterates over every artifact in the trail, looks up its `snyk-container-scan` attestation, and checks whether any result has a non-zero `high_count`. + +Use `--attestations` to enrich only the snyk data (faster than fetching all attestation details): + ```shell {.command} kosli evaluate trail \ - --policy pr-approved-single.rego \ + --policy snyk-no-high-vulns.rego \ --org cyber-dojo \ --flow dashboard-ci \ - 9978a1ca82c273a68afaa85fc37dd60d1e394f84 + --attestations dashboard.snyk-container-scan \ + 44ca5fa2630947cf375fdbda10972a4bedaaaba3 ``` ```plaintext {.light-console} -RESULT: DENIED -VIOLATIONS: trail '9978a1ca82c273a68afaa85fc37dd60d1e394f84': pull-request https://github.com/cyber-dojo/dashboard/pull/344 has no approvers +RESULT: ALLOWED ``` +The trail has zero high-severity vulnerabilities, so the policy allows it. + {{}} -When writing a policy for `kosli evaluate trail`, reference `input.trail` (a single object) instead of `input.trails` (an array). You can write one policy that handles both by checking for both keys, or keep separate policies for each command. +When writing a policy for `kosli evaluate trail`, reference `input.trail` (a single object). For `kosli evaluate trails`, reference `input.trails` (an array). The data shapes differ, so use separate policies for each command. {{}} ## Step 5: Explore the policy input with --show-input @@ -148,24 +162,26 @@ When writing policies, it helps to see exactly what data is available. Use `--sh ```shell {.command} kosli evaluate trail \ - --policy pr-approved-single.rego \ + --policy snyk-no-high-vulns.rego \ --org cyber-dojo \ --flow dashboard-ci \ + --attestations dashboard.snyk-container-scan \ --show-input \ --output json \ - 9978a1ca82c273a68afaa85fc37dd60d1e394f84 + 44ca5fa2630947cf375fdbda10972a4bedaaaba3 ``` This outputs the evaluation result along with the complete `input` object. You can pipe it through `jq` to explore the structure: ```shell {.command} kosli evaluate trail \ - --policy pr-approved-single.rego \ + --policy snyk-no-high-vulns.rego \ --org cyber-dojo \ --flow dashboard-ci \ + --attestations dashboard.snyk-container-scan \ --show-input \ --output json \ - 9978a1ca82c273a68afaa85fc37dd60d1e394f84 2>/dev/null | jq '.input.trail.compliance_status | keys' + 44ca5fa2630947cf375fdbda10972a4bedaaaba3 2>/dev/null | jq '.input.trail.compliance_status | keys' ``` ```plaintext {.light-console} @@ -217,7 +233,7 @@ you'd set in your CI/CD pipeline: # Run the evaluation and save the full JSON report to a file # (|| true prevents the step from failing when the policy denies) kosli evaluate trail "$TRAIL_NAME" \ - --policy pr-approved-single.rego \ + --policy my-policy.rego \ --org "$KOSLI_ORG" \ --flow "$FLOW_NAME" \ --show-input \ @@ -236,7 +252,7 @@ kosli attest generic \ --trail "$TRAIL_NAME" \ --org "$KOSLI_ORG" \ --compliant="$is_compliant" \ - --attachments pr-approved-single.rego,eval-report.json \ + --attachments my-policy.rego,eval-report.json \ --user-data eval-violations.json ``` From e7c82508c8e5e1fdb7dee43c9a788502075e4013 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Fri, 6 Mar 2026 17:37:35 +0000 Subject: [PATCH 3/5] publish evaluate trails tutorial (remove draft status) Co-Authored-By: Claude Opus 4.6 --- docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md index 46a8eb25a..632882bcd 100644 --- a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md +++ b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md @@ -2,7 +2,6 @@ title: "Evaluate trails with OPA policies" bookCollapseSection: false weight: 509 -draft: true summary: "Learn how to use kosli evaluate trail and kosli evaluate trails to check your Kosli trails against custom OPA/Rego policies. This tutorial walks through writing a policy that verifies pull requests have been approved." --- From 9cec5316fece2d60794c30d9cdf51479f76c228b Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Mon, 9 Mar 2026 13:17:50 +0000 Subject: [PATCH 4/5] address PR #687 review feedback: clarify attestations format, add placeholder note, comment backtick pattern Co-Authored-By: Claude Opus 4.6 --- cmd/kosli/evaluate.go | 2 ++ .../content/tutorials/evaluate_trails_with_opa.md | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/kosli/evaluate.go b/cmd/kosli/evaluate.go index 39c013f1d..c06895299 100644 --- a/cmd/kosli/evaluate.go +++ b/cmd/kosli/evaluate.go @@ -8,6 +8,8 @@ import ( const evaluateShortDesc = `Evaluate Kosli trail data against OPA/Rego policies.` +// Backtick breaks (`"` + "`x`" + `"`) are needed to embed markdown +// inline code spans inside raw string literals. const evaluateLongDesc = evaluateShortDesc + ` Fetch trail data from Kosli and evaluate it against custom policies written in Rego, the policy language used by Open Policy Agent (OPA). diff --git a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md index 632882bcd..a49853b7e 100644 --- a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md +++ b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md @@ -134,7 +134,8 @@ allow if { This policy iterates over every artifact in the trail, looks up its `snyk-container-scan` attestation, and checks whether any result has a non-zero `high_count`. -Use `--attestations` to enrich only the snyk data (faster than fetching all attestation details): +Use `--attestations` to enrich only the snyk data (faster than fetching all attestation details). +The value uses the format `artifact-name.attestation-type` — here, `dashboard` is the artifact name and `snyk-container-scan` is the attestation type: ```shell {.command} kosli evaluate trail \ @@ -226,7 +227,8 @@ audit record in Kosli that captures the policy, the full evaluation report, and violations. This step requires write access to your Kosli org. The examples below use variables -you'd set in your CI/CD pipeline: +you'd set in your CI/CD pipeline. +In your own pipeline you'd use your own policy file — here we use `my-policy.rego` as a placeholder: ```shell {.command} # Run the evaluation and save the full JSON report to a file From 3ad33c0bb378726dfd39735bc5786212c016a112 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Mon, 9 Mar 2026 13:23:07 +0000 Subject: [PATCH 5/5] Fix type -> name error in tutorial --- .../tutorials/evaluate_trails_with_opa.md | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md index a49853b7e..628faf800 100644 --- a/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md +++ b/docs.kosli.com/content/tutorials/evaluate_trails_with_opa.md @@ -18,6 +18,7 @@ To follow this tutorial, you need to: * [Install Kosli CLI](/getting_started/install/). * [Get a Kosli API token](/getting_started/service-accounts/). * Set the `KOSLI_API_TOKEN` environment variable to your token: + ```shell {.command} export KOSLI_API_TOKEN= ``` @@ -51,18 +52,18 @@ allow if { Let's break down what this policy does: -- **`package policy`** — every evaluate policy must use the `policy` package. -- **`import rego.v1`** — use Rego v1 syntax (the `if`/`contains` keywords). -- **`default allow = false`** — trails are denied unless explicitly allowed. -- **`violations`** — a set of messages describing why the policy failed. The rule iterates over trails, then over pull requests within the `pull-request` attestation, looking for PRs where `approvers` is empty. -- **`allow`** — trails are allowed only when there are no violations. +* **`package policy`** — every evaluate policy must use the `policy` package. +* **`import rego.v1`** — use Rego v1 syntax (the `if`/`contains` keywords). +* **`default allow = false`** — trails are denied unless explicitly allowed. +* **`violations`** — a set of messages describing why the policy failed. The rule iterates over trails, then over pull requests within the `pull-request` attestation, looking for PRs where `approvers` is empty. +* **`allow`** — trails are allowed only when there are no violations. {{}} **Policy contract** — these are Kosli-specific conventions, not OPA built-ins: -- **`package policy`** — required. Kosli queries `data.policy.*` to find your rules. -- **`allow`** — required. Must evaluate to a **boolean**. Kosli exits with code 0 when `true`, code 1 when `false`. -- **`violations`** — optional but recommended. Must be a **set of strings**, where each string is a human-readable reason the policy failed. Kosli displays these when `allow` is `false`. +* **`package policy`** — required. Kosli queries `data.policy.*` to find your rules. +* **`allow`** — required. Must evaluate to a **boolean**. Kosli exits with code 0 when `true`, code 1 when `false`. +* **`violations`** — optional but recommended. Must be a **set of strings**, where each string is a human-readable reason the policy failed. Kosli displays these when `allow` is `false`. {{}} ## Step 3: Evaluate multiple trails @@ -135,7 +136,7 @@ allow if { This policy iterates over every artifact in the trail, looks up its `snyk-container-scan` attestation, and checks whether any result has a non-zero `high_count`. Use `--attestations` to enrich only the snyk data (faster than fetching all attestation details). -The value uses the format `artifact-name.attestation-type` — here, `dashboard` is the artifact name and `snyk-container-scan` is the attestation type: +The value uses the format `artifact-name.attestation-type` — here, `dashboard` is the artifact name and `snyk-container-scan` is the attestation name: ```shell {.command} kosli evaluate trail \ @@ -259,12 +260,12 @@ kosli attest generic \ This creates a generic attestation on the trail with: -- **`--compliant`** set based on whether the policy allowed or denied — read directly +* **`--compliant`** set based on whether the policy allowed or denied — read directly from the JSON report rather than relying on the exit code, which avoids issues with `set -e` in CI environments like GitHub Actions -- **`--attachments`** containing the Rego policy (for reproducibility) and the full +* **`--attachments`** containing the Rego policy (for reproducibility) and the full JSON evaluation report (including the input data the policy evaluated) -- **`--user-data`** containing the violations, which appear in the Kosli UI as +* **`--user-data`** containing the violations, which appear in the Kosli UI as structured metadata on the attestation {{}}