From b0f743d2ca8b06ac60f5b1f93ebf73b8b33471bb Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 7 Apr 2026 13:19:31 +0300 Subject: [PATCH 01/12] feat(query): add QueryLog collector to QueryLogger Add QueryLogEntry struct and QueryLog collector that can be attached to a context via WithQueryLog(). When present, QueryTimer.End() appends query metadata (name, args, count, duration, error, summary) to the collector for downstream consumption. --- query/config_tree.go | 17 ++++++- query/query_logger.go | 102 +++++++++++++++++++++++++++++++++++------- 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/query/config_tree.go b/query/config_tree.go index d371a19e0..ee9b2c854 100644 --- a/query/config_tree.go +++ b/query/config_tree.go @@ -180,9 +180,22 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi parentID := lo.FromPtr(c.ParentID) if parent, ok := nodes[parentID]; ok { parent.children = append(parent.children, nodes[c.ID]) - } else { - targetNode.children = append(targetNode.children, nodes[c.ID]) + continue + } + if c.Path != "" { + segments := strings.Split(c.Path, ".") + if len(segments) >= 2 { + if parentStr := segments[len(segments)-2]; parentStr != "" { + if pid, err := uuid.Parse(parentStr); err == nil { + if parent, ok := nodes[pid]; ok { + parent.children = append(parent.children, nodes[c.ID]) + continue + } + } + } + } } + targetNode.children = append(targetNode.children, nodes[c.ID]) } parentIDs := make(map[uuid.UUID]bool, len(parents)) diff --git a/query/query_logger.go b/query/query_logger.go index a30ba3591..26cd693b6 100644 --- a/query/query_logger.go +++ b/query/query_logger.go @@ -3,7 +3,9 @@ package query import ( "fmt" "reflect" + "slices" "strings" + "sync" "time" clickyapi "github.com/flanksource/clicky/api" @@ -11,16 +13,60 @@ import ( "github.com/flanksource/duty/context" ) +type QueryLogEntry struct { + Name string `json:"name"` + Args string `json:"args,omitempty"` + Count int `json:"count"` + Duration int64 `json:"duration"` + Error string `json:"error,omitempty"` + Summary string `json:"summary,omitempty"` +} + +type QueryLog struct { + mu sync.Mutex + entries []QueryLogEntry +} + +func (q *QueryLog) Append(e QueryLogEntry) { + q.mu.Lock() + defer q.mu.Unlock() + q.entries = append(q.entries, e) +} + +func (q *QueryLog) Entries() []QueryLogEntry { + q.mu.Lock() + defer q.mu.Unlock() + return slices.Clone(q.entries) +} + +type queryLogKey struct{} + +func WithQueryLog(ctx context.Context) (context.Context, *QueryLog) { + log := &QueryLog{} + return ctx.WithValue(queryLogKey{}, log), log +} + +func GetQueryLog(ctx context.Context) *QueryLog { + if v, ok := ctx.Value(queryLogKey{}).(*QueryLog); ok { + return v + } + return nil +} + type QueryLogger struct { - logger logger.Verbose + logger logger.Verbose + queryLog *QueryLog } type QueryTimer struct { - logger logger.Verbose - label clickyapi.Text - start time.Time - results any - ended bool + logger logger.Verbose + queryLog *QueryLog + name string + args string + label clickyapi.Text + start time.Time + results any + ended bool } func NewQueryLogger(ctx context.Context) QueryLogger { @@ -28,14 +74,16 @@ func NewQueryLogger(ctx context.Context) QueryLogger { if ctx.Properties().On(false, "query.log") { l = ctx.Logger.V(0) } - return QueryLogger{logger: l} + return QueryLogger{logger: l, queryLog: GetQueryLog(ctx)} } func (q QueryLogger) Start(entity string) *QueryTimer { return &QueryTimer{ - logger: q.logger, - label: clickyapi.Text{Content: "[" + entity + "]", Style: "text-blue-600 font-bold"}, - start: time.Now(), + logger: q.logger, + queryLog: q.queryLog, + name: entity, + label: clickyapi.Text{Content: "[" + entity + "]", Style: "text-blue-600 font-bold"}, + start: time.Now(), } } @@ -46,6 +94,10 @@ func (t *QueryTimer) Arg(key string, value any) *QueryTimer { } t.label = t.label.AddText(fmt.Sprintf(" %s=", key), "text-gray-500"). AddText(s) + if t.args != "" { + t.args += " " + } + t.args += fmt.Sprintf("%s=%s", key, s) return t } @@ -59,15 +111,23 @@ func (t *QueryTimer) End(err *error) { return } t.ended = true - if !t.logger.Enabled() { - return - } elapsed := time.Since(t.start) + + var entry QueryLogEntry + if t.queryLog != nil { + entry.Name = t.name + entry.Args = t.args + entry.Duration = elapsed.Milliseconds() + } + label := t.label.AddText(" => ", "text-gray-400") if err != nil && *err != nil { label = label.AddText(fmt.Sprintf("error: %v", *err), "text-red-600") + if t.queryLog != nil { + entry.Error = (*err).Error() + } } else if t.results != nil { count := sliceLen(t.results) countStyle := "text-green-600" @@ -75,13 +135,25 @@ func (t *QueryTimer) End(err *error) { countStyle = "text-red-600" } label = label.AddText(fmt.Sprintf("%d", count), countStyle) - label = label.AddText(summaryText(t.results, count), "text-gray-400") + summary := summaryText(t.results, count) + label = label.AddText(summary, "text-gray-400") + if t.queryLog != nil { + entry.Count = count + entry.Summary = summary + } } else { label = label.AddText("timed out", "text-yellow-600") } label = label.AddText(fmt.Sprintf(" in %dms", elapsed.Milliseconds()), "text-gray-400") - t.logger.Infof("%s", label.ANSI()) + + if t.queryLog != nil { + t.queryLog.Append(entry) + } + + if t.logger.Enabled() { + t.logger.Infof("%s", label.ANSI()) + } } func sliceLen(v any) int { From 698c6864e4b159046639d75c879be5d27f1f6996 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 7 Apr 2026 14:19:49 +0300 Subject: [PATCH 02/12] feat(types): add typed change details for config changes Introduces strongly-typed structs for different change detail categories (deployment, backup, pipeline, scaling, etc.) with automatic kind injection during JSON marshaling. Adds constants for all change type values and generates OpenAPI schema for validation. --- hack/generate-schemas/main.go | 1 + schema/openapi/change-types.schema.json | 350 +++++++++++++++++++++++ tests/fixtures/dummy/all.go | 8 +- tests/fixtures/dummy/application_data.go | 38 +-- tests/fixtures/dummy/config_changes.go | 21 +- types/config_changes.go | 275 ++++++++++++++++++ types/config_changes_test.go | 96 +++++++ 7 files changed, 756 insertions(+), 33 deletions(-) create mode 100644 schema/openapi/change-types.schema.json create mode 100644 types/config_changes.go create mode 100644 types/config_changes_test.go diff --git a/hack/generate-schemas/main.go b/hack/generate-schemas/main.go index 721c14640..24629a3f9 100644 --- a/hack/generate-schemas/main.go +++ b/hack/generate-schemas/main.go @@ -13,6 +13,7 @@ import ( var schemas = map[string]any{ "resource_selector": &types.ResourceSelector{}, "resource_selectors": &[]types.ResourceSelector{}, + "change-types": &types.ConfigChangeDetailsSchema{}, } var generateSchema = &cobra.Command{ diff --git a/schema/openapi/change-types.schema.json b/schema/openapi/change-types.schema.json new file mode 100644 index 000000000..c06babeb0 --- /dev/null +++ b/schema/openapi/change-types.schema.json @@ -0,0 +1,350 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/flanksource/duty/types/config-change-details-schema", + "$ref": "#/$defs/ConfigChangeDetailsSchema", + "$defs": { + "ApprovalDetails": { + "properties": { + "playbook_id": { + "type": "string" + }, + "run_id": { + "type": "string" + }, + "approved_by": { + "type": "string" + }, + "rejected_by": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "BackupDetails": { + "properties": { + "status": { + "type": "string" + }, + "size": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "backup_type": { + "type": "string" + }, + "target": { + "type": "string" + }, + "snapshot_id": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "CertificateDetails": { + "properties": { + "subject": { + "type": "string" + }, + "issuer": { + "type": "string" + }, + "not_before": { + "type": "string" + }, + "not_after": { + "type": "string" + }, + "serial": { + "type": "string" + }, + "dns_names": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "ConfigChangeDetailsSchema": { + "properties": { + "UserChange/v1": { + "$ref": "#/$defs/UserChangeDetails" + }, + "Screenshot/v1": { + "$ref": "#/$defs/ScreenshotDetails" + }, + "PermissionChange/v1": { + "$ref": "#/$defs/PermissionChangeDetails" + }, + "Deployment/v1": { + "$ref": "#/$defs/DeploymentDetails" + }, + "Promotion/v1": { + "$ref": "#/$defs/PromotionDetails" + }, + "Approval/v1": { + "$ref": "#/$defs/ApprovalDetails" + }, + "Rollback/v1": { + "$ref": "#/$defs/RollbackDetails" + }, + "Backup/v1": { + "$ref": "#/$defs/BackupDetails" + }, + "PlaybookExecution/v1": { + "$ref": "#/$defs/PlaybookExecutionDetails" + }, + "Scaling/v1": { + "$ref": "#/$defs/ScalingDetails" + }, + "Certificate/v1": { + "$ref": "#/$defs/CertificateDetails" + }, + "CostChange/v1": { + "$ref": "#/$defs/CostChangeDetails" + }, + "PipelineRun/v1": { + "$ref": "#/$defs/PipelineRunDetails" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ConfigChangeDetailsSchema is a union type used for JSON schema generation.\nIt is never instantiated at runtime." + }, + "CostChangeDetails": { + "properties": { + "previous_cost": { + "type": "number" + }, + "new_cost": { + "type": "number" + }, + "currency": { + "type": "string" + }, + "period": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "DeploymentDetails": { + "properties": { + "previous_image": { + "type": "string" + }, + "new_image": { + "type": "string" + }, + "container": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "strategy": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "PermissionChangeDetails": { + "properties": { + "user_id": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "group_name": { + "type": "string" + }, + "role_id": { + "type": "string" + }, + "role_name": { + "type": "string" + }, + "role_type": { + "type": "string" + }, + "scope": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "PipelineRunDetails": { + "properties": { + "pipeline_id": { + "type": "string" + }, + "pipeline_name": { + "type": "string" + }, + "run_id": { + "type": "string" + }, + "run_number": { + "type": "integer" + }, + "branch": { + "type": "string" + }, + "status": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "error": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "PlaybookExecutionDetails": { + "properties": { + "playbook_id": { + "type": "string" + }, + "playbook_name": { + "type": "string" + }, + "run_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "error": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "PromotionDetails": { + "properties": { + "from_environment": { + "type": "string" + }, + "to_environment": { + "type": "string" + }, + "version": { + "type": "string" + }, + "artifact": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "RollbackDetails": { + "properties": { + "from_version": { + "type": "string" + }, + "to_version": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "trigger": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "ScalingDetails": { + "properties": { + "from_replicas": { + "type": "integer" + }, + "to_replicas": { + "type": "integer" + }, + "resource_type": { + "type": "string" + }, + "trigger": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + }, + "ScreenshotDetails": { + "properties": { + "artifact_id": { + "type": "string" + }, + "url": { + "type": "string" + }, + "content_type": { + "type": "string" + }, + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object" + }, + "UserChangeDetails": { + "properties": { + "user_id": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "user_email": { + "type": "string" + }, + "user_type": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "group_name": { + "type": "string" + }, + "tenant": { + "type": "string" + } + }, + "additionalProperties": false, + "type": "object" + } + } +} \ No newline at end of file diff --git a/tests/fixtures/dummy/all.go b/tests/fixtures/dummy/all.go index b2ffd0b6e..cf90a5fdb 100644 --- a/tests/fixtures/dummy/all.go +++ b/tests/fixtures/dummy/all.go @@ -1021,26 +1021,26 @@ func GenerateDynamicDummyData(db *gorm.DB) DummyData { var EKSClusterCreateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, CreatedAt: &DummyYearOldDate, } var EKSClusterUpdateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, } var EKSClusterDeleteChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "DELETE", + ChangeType: types.ChangeTypeDelete, } var KubernetesNodeAChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: KubernetesNodeA.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, } var configChanges = []models.ConfigChange{ diff --git a/tests/fixtures/dummy/application_data.go b/tests/fixtures/dummy/application_data.go index 5ffeaa0db..12478598b 100644 --- a/tests/fixtures/dummy/application_data.go +++ b/tests/fixtures/dummy/application_data.go @@ -80,7 +80,7 @@ var RDSBackupChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: RDSInstance.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "AWS", Details: types.JSON(`{"status":"success","size":"4.2GB"}`), CreatedAt: &appT48h, @@ -88,7 +88,7 @@ var RDSBackupChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: RDSInstance.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "AWS", Details: types.JSON(`{"status":"success","size":"4.3GB"}`), CreatedAt: &appT24h, @@ -96,7 +96,7 @@ var RDSBackupChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: RDSInstance.ID.String(), - ChangeType: "BackupRestored", + ChangeType: types.ChangeTypeBackupRestored, Source: "AWS", Details: types.JSON(`{"status":"success"}`), CreatedAt: &appT12h, @@ -107,7 +107,7 @@ var DeploymentDiffChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: IncidentCommanderDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "low", Summary: "image updated: v1.2.3 -> v1.2.4", @@ -116,7 +116,7 @@ var DeploymentDiffChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: IncidentCommanderDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "info", Summary: "replicas scaled: 2 -> 3", @@ -274,7 +274,7 @@ var KubernetesAppDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("a1b2c3d4-e5f6-7890-abcd-000000000010").String(), ConfigID: KubernetesAppDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "low", Summary: "image updated: v2.0.9 -> v2.1.0", @@ -283,7 +283,7 @@ var KubernetesAppDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("a1b2c3d4-e5f6-7890-abcd-000000000011").String(), ConfigID: KubernetesAppDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "info", Summary: "replicas scaled: 2 -> 3", @@ -292,7 +292,7 @@ var KubernetesAppDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("a1b2c3d4-e5f6-7890-abcd-000000000012").String(), ConfigID: KubernetesAppIngress.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "medium", Summary: "TLS certificate renewed", @@ -367,7 +367,7 @@ var MSSQLBackupChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000010").String(), ConfigID: MSSQLProdDatabase.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "mssql", Details: types.JSON(`{"status":"success","size":"12.4GB","type":"FULL"}`), CreatedAt: &appT48h, @@ -375,7 +375,7 @@ var MSSQLBackupChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000011").String(), ConfigID: MSSQLProdDatabase.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "mssql", Details: types.JSON(`{"status":"success","size":"12.7GB","type":"FULL"}`), CreatedAt: &appT24h, @@ -386,7 +386,7 @@ var MSSQLDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000012").String(), ConfigID: MSSQLProdDatabase.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "mssql", Severity: "medium", Summary: "schema migration: added column orders.fulfilled_at", @@ -395,7 +395,7 @@ var MSSQLDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000013").String(), ConfigID: MSSQLServer.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "mssql", Severity: "low", Summary: "server collation updated", @@ -530,7 +530,7 @@ var PopAPIRepoDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("c3d4e5f6-a7b8-9012-cdef-000000000010").String(), ConfigID: PopAPIRepo.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "github", Severity: "info", Summary: "PR #124 merged: add connection pooling", @@ -539,7 +539,7 @@ var PopAPIRepoDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("c3d4e5f6-a7b8-9012-cdef-000000000011").String(), ConfigID: PopAPIRepo.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "github", Severity: "low", Summary: "tag pushed: v0.9.3", @@ -551,7 +551,7 @@ var PopAPIDBChanges = []models.ConfigChange{ { ID: uuid.MustParse("c3d4e5f6-a7b8-9012-cdef-000000000012").String(), ConfigID: PopAPIDatabase.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "postgresql", Details: types.JSON(`{"status":"success","size":"512MB"}`), CreatedAt: &appT24h, @@ -612,7 +612,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000010").String(), ConfigID: AzDOBuildPipeline.ID.String(), - ChangeType: "PipelineRunStarted", + ChangeType: types.ChangeTypePipelineRunStarted, Source: "azuredevops", Severity: "info", Summary: "build #88 started on main", @@ -621,7 +621,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000011").String(), ConfigID: AzDOBuildPipeline.ID.String(), - ChangeType: "PipelineRunCompleted", + ChangeType: types.ChangeTypePipelineRunCompleted, Source: "azuredevops", Severity: "info", Summary: "build #88 succeeded in 4m32s", @@ -630,7 +630,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000012").String(), ConfigID: AzDOBuildPipeline.ID.String(), - ChangeType: "PipelineRunFailed", + ChangeType: types.ChangeTypePipelineRunFailed, Source: "azuredevops", Severity: "high", Summary: "build #87 failed: test stage timed out", @@ -639,7 +639,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000013").String(), ConfigID: AzDOReleasePipeline.ID.String(), - ChangeType: "PipelineRunCompleted", + ChangeType: types.ChangeTypePipelineRunCompleted, Source: "azuredevops", Severity: "info", Summary: "release #12 deployed to production", diff --git a/tests/fixtures/dummy/config_changes.go b/tests/fixtures/dummy/config_changes.go index d2a2fe0fd..08e3c3016 100644 --- a/tests/fixtures/dummy/config_changes.go +++ b/tests/fixtures/dummy/config_changes.go @@ -7,12 +7,13 @@ import ( "github.com/samber/lo" "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" ) var EKSClusterCreateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, CreatedAt: &DummyYearOldDate, Severity: models.SeverityMedium, Source: "CloudTrail", @@ -24,7 +25,7 @@ var EKSClusterCreateChange = models.ConfigChange{ var EKSClusterUpdateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 24)), Severity: models.SeverityLow, Source: "CloudTrail", @@ -36,7 +37,7 @@ var EKSClusterUpdateChange = models.ConfigChange{ var EKSClusterDeleteChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "DELETE", + ChangeType: types.ChangeTypeDelete, CreatedAt: &DummyNow, Severity: models.SeverityHigh, Source: "CloudTrail", @@ -48,7 +49,7 @@ var EKSClusterDeleteChange = models.ConfigChange{ var KubernetesNodeAChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: KubernetesNodeA.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, CreatedAt: &DummyYearOldDate, Severity: models.SeverityInfo, Source: "Kubernetes", @@ -61,7 +62,7 @@ var KubernetesNodeAChange = models.ConfigChange{ var NginxHelmReleaseUpgradeV1 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: NginxHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 72)), // 3 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -73,7 +74,7 @@ var NginxHelmReleaseUpgradeV1 = models.ConfigChange{ var NginxHelmReleaseUpgradeV2 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: NginxHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 48)), // 2 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -85,7 +86,7 @@ var NginxHelmReleaseUpgradeV2 = models.ConfigChange{ var NginxHelmReleaseUpgradeV3 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: NginxHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 24)), // 1 day ago Severity: models.SeverityInfo, Source: "Flux", @@ -98,7 +99,7 @@ var NginxHelmReleaseUpgradeV3 = models.ConfigChange{ var RedisHelmReleaseUpgradeV1 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: RedisHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 96)), // 4 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -110,7 +111,7 @@ var RedisHelmReleaseUpgradeV1 = models.ConfigChange{ var RedisHelmReleaseUpgradeV2 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: RedisHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 60)), // 2.5 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -122,7 +123,7 @@ var RedisHelmReleaseUpgradeV2 = models.ConfigChange{ var RedisHelmReleaseUpgradeV3 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: RedisHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 36)), // 1.5 days ago Severity: models.SeverityInfo, Source: "Flux", diff --git a/types/config_changes.go b/types/config_changes.go new file mode 100644 index 000000000..d47f3aded --- /dev/null +++ b/types/config_changes.go @@ -0,0 +1,275 @@ +package types + +import "encoding/json" + +// ConfigChangeDetail is implemented by all typed detail structs +// for the ConfigChange.Details field. +type ConfigChangeDetail interface { + Kind() string +} + +// Change type constants. +const ( + ChangeTypeCreate = "CREATE" + ChangeTypeUpdate = "UPDATE" + ChangeTypeDelete = "DELETE" + ChangeTypeDiff = "diff" + + ChangeTypeUserCreated = "UserCreated" + ChangeTypeUserDeleted = "UserDeleted" + ChangeTypeGroupMemberAdded = "GroupMemberAdded" + ChangeTypeGroupMemberRemoved = "GroupMemberRemoved" + + ChangeTypeScreenshot = "Screenshot" + + ChangeTypePermissionAdded = "PermissionAdded" + ChangeTypePermissionRemoved = "PermissionRemoved" + + ChangeTypeDeployment = "Deployment" + ChangeTypePromotion = "Promotion" + ChangeTypeApproved = "Approved" + ChangeTypeRejected = "Rejected" + ChangeTypeRollback = "Rollback" + + ChangeTypeBackupCompleted = "BackupCompleted" + ChangeTypeBackupRestored = "BackupRestored" + ChangeTypeBackupFailed = "BackupFailed" + + ChangeTypePipelineRunStarted = "PipelineRunStarted" + ChangeTypePipelineRunCompleted = "PipelineRunCompleted" + ChangeTypePipelineRunFailed = "PipelineRunFailed" + + ChangeTypeScaling = "Scaling" + + ChangeTypeCertificateRenewed = "CertificateRenewed" + ChangeTypeCertificateExpired = "CertificateExpired" + + ChangeTypeCostChange = "CostChange" + + ChangeTypePlaybookStarted = "PlaybookStarted" + ChangeTypePlaybookCompleted = "PlaybookCompleted" + ChangeTypePlaybookFailed = "PlaybookFailed" + + ChangeTypeRunInstances = "RunInstances" + ChangeTypeRegisterNode = "RegisterNode" + ChangeTypePulled = "Pulled" +) + +func marshalWithKind(kind string, v any) ([]byte, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + if len(data) > 2 { + return append([]byte(`{"kind":"`+kind+`",`), data[1:]...), nil + } + return []byte(`{"kind":"` + kind + `"}`), nil +} + +// ConfigChangeDetailsSchema is a union type used for JSON schema generation. +// It is never instantiated at runtime. +type ConfigChangeDetailsSchema struct { + UserChange *UserChangeDetails `json:"UserChange/v1,omitempty"` + Screenshot *ScreenshotDetails `json:"Screenshot/v1,omitempty"` + PermissionChange *PermissionChangeDetails `json:"PermissionChange/v1,omitempty"` + Deployment *DeploymentDetails `json:"Deployment/v1,omitempty"` + Promotion *PromotionDetails `json:"Promotion/v1,omitempty"` + Approval *ApprovalDetails `json:"Approval/v1,omitempty"` + Rollback *RollbackDetails `json:"Rollback/v1,omitempty"` + Backup *BackupDetails `json:"Backup/v1,omitempty"` + PlaybookExecution *PlaybookExecutionDetails `json:"PlaybookExecution/v1,omitempty"` + Scaling *ScalingDetails `json:"Scaling/v1,omitempty"` + Certificate *CertificateDetails `json:"Certificate/v1,omitempty"` + CostChange *CostChangeDetails `json:"CostChange/v1,omitempty"` + PipelineRun *PipelineRunDetails `json:"PipelineRun/v1,omitempty"` +} + +type UserChangeDetails struct { + UserID string `json:"user_id,omitempty"` + UserName string `json:"user_name,omitempty"` + UserEmail string `json:"user_email,omitempty"` + UserType string `json:"user_type,omitempty"` + GroupID string `json:"group_id,omitempty"` + GroupName string `json:"group_name,omitempty"` + Tenant string `json:"tenant,omitempty"` +} + +func (d UserChangeDetails) Kind() string { return "UserChange/v1" } +func (d UserChangeDetails) MarshalJSON() ([]byte, error) { + type raw UserChangeDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type ScreenshotDetails struct { + ArtifactID string `json:"artifact_id,omitempty"` + URL string `json:"url,omitempty"` + ContentType string `json:"content_type,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` +} + +func (d ScreenshotDetails) Kind() string { return "Screenshot/v1" } +func (d ScreenshotDetails) MarshalJSON() ([]byte, error) { + type raw ScreenshotDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type PermissionChangeDetails struct { + UserID string `json:"user_id,omitempty"` + UserName string `json:"user_name,omitempty"` + GroupID string `json:"group_id,omitempty"` + GroupName string `json:"group_name,omitempty"` + RoleID string `json:"role_id,omitempty"` + RoleName string `json:"role_name,omitempty"` + RoleType string `json:"role_type,omitempty"` + Scope string `json:"scope,omitempty"` +} + +func (d PermissionChangeDetails) Kind() string { return "PermissionChange/v1" } +func (d PermissionChangeDetails) MarshalJSON() ([]byte, error) { + type raw PermissionChangeDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type DeploymentDetails struct { + PreviousImage string `json:"previous_image,omitempty"` + NewImage string `json:"new_image,omitempty"` + Container string `json:"container,omitempty"` + Namespace string `json:"namespace,omitempty"` + Strategy string `json:"strategy,omitempty"` +} + +func (d DeploymentDetails) Kind() string { return "Deployment/v1" } +func (d DeploymentDetails) MarshalJSON() ([]byte, error) { + type raw DeploymentDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type PromotionDetails struct { + FromEnvironment string `json:"from_environment,omitempty"` + ToEnvironment string `json:"to_environment,omitempty"` + Version string `json:"version,omitempty"` + Artifact string `json:"artifact,omitempty"` +} + +func (d PromotionDetails) Kind() string { return "Promotion/v1" } +func (d PromotionDetails) MarshalJSON() ([]byte, error) { + type raw PromotionDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type ApprovalDetails struct { + PlaybookID string `json:"playbook_id,omitempty"` + RunID string `json:"run_id,omitempty"` + ApprovedBy string `json:"approved_by,omitempty"` + RejectedBy string `json:"rejected_by,omitempty"` + Reason string `json:"reason,omitempty"` +} + +func (d ApprovalDetails) Kind() string { return "Approval/v1" } +func (d ApprovalDetails) MarshalJSON() ([]byte, error) { + type raw ApprovalDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type RollbackDetails struct { + FromVersion string `json:"from_version,omitempty"` + ToVersion string `json:"to_version,omitempty"` + Reason string `json:"reason,omitempty"` + Trigger string `json:"trigger,omitempty"` +} + +func (d RollbackDetails) Kind() string { return "Rollback/v1" } +func (d RollbackDetails) MarshalJSON() ([]byte, error) { + type raw RollbackDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type BackupDetails struct { + Status string `json:"status,omitempty"` + Size string `json:"size,omitempty"` + Duration string `json:"duration,omitempty"` + BackupType string `json:"backup_type,omitempty"` + Target string `json:"target,omitempty"` + SnapshotID string `json:"snapshot_id,omitempty"` +} + +func (d BackupDetails) Kind() string { return "Backup/v1" } +func (d BackupDetails) MarshalJSON() ([]byte, error) { + type raw BackupDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type PlaybookExecutionDetails struct { + PlaybookID string `json:"playbook_id,omitempty"` + PlaybookName string `json:"playbook_name,omitempty"` + RunID string `json:"run_id,omitempty"` + Status string `json:"status,omitempty"` + Duration string `json:"duration,omitempty"` + Error string `json:"error,omitempty"` +} + +func (d PlaybookExecutionDetails) Kind() string { return "PlaybookExecution/v1" } +func (d PlaybookExecutionDetails) MarshalJSON() ([]byte, error) { + type raw PlaybookExecutionDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type ScalingDetails struct { + FromReplicas int `json:"from_replicas,omitempty"` + ToReplicas int `json:"to_replicas,omitempty"` + ResourceType string `json:"resource_type,omitempty"` + Trigger string `json:"trigger,omitempty"` +} + +func (d ScalingDetails) Kind() string { return "Scaling/v1" } +func (d ScalingDetails) MarshalJSON() ([]byte, error) { + type raw ScalingDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type CertificateDetails struct { + Subject string `json:"subject,omitempty"` + Issuer string `json:"issuer,omitempty"` + NotBefore string `json:"not_before,omitempty"` + NotAfter string `json:"not_after,omitempty"` + Serial string `json:"serial,omitempty"` + DNSNames string `json:"dns_names,omitempty"` +} + +func (d CertificateDetails) Kind() string { return "Certificate/v1" } +func (d CertificateDetails) MarshalJSON() ([]byte, error) { + type raw CertificateDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type CostChangeDetails struct { + PreviousCost float64 `json:"previous_cost,omitempty"` + NewCost float64 `json:"new_cost,omitempty"` + Currency string `json:"currency,omitempty"` + Period string `json:"period,omitempty"` + Reason string `json:"reason,omitempty"` +} + +func (d CostChangeDetails) Kind() string { return "CostChange/v1" } +func (d CostChangeDetails) MarshalJSON() ([]byte, error) { + type raw CostChangeDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type PipelineRunDetails struct { + PipelineID string `json:"pipeline_id,omitempty"` + PipelineName string `json:"pipeline_name,omitempty"` + RunID string `json:"run_id,omitempty"` + RunNumber int `json:"run_number,omitempty"` + Branch string `json:"branch,omitempty"` + Status string `json:"status,omitempty"` + Duration string `json:"duration,omitempty"` + Error string `json:"error,omitempty"` +} + +func (d PipelineRunDetails) Kind() string { return "PipelineRun/v1" } +func (d PipelineRunDetails) MarshalJSON() ([]byte, error) { + type raw PipelineRunDetails + return marshalWithKind(d.Kind(), raw(d)) +} diff --git a/types/config_changes_test.go b/types/config_changes_test.go new file mode 100644 index 000000000..f072d7144 --- /dev/null +++ b/types/config_changes_test.go @@ -0,0 +1,96 @@ +package types + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ConfigChangeDetails", func() { + It("should inject kind into DeploymentDetails", func() { + d := DeploymentDetails{PreviousImage: "v1.2.3", NewImage: "v1.2.4", Container: "app"} + data, err := json.Marshal(d) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("Deployment/v1")) + Expect(m["previous_image"]).To(Equal("v1.2.3")) + Expect(m["new_image"]).To(Equal("v1.2.4")) + Expect(m["container"]).To(Equal("app")) + }) + + It("should inject kind into BackupDetails", func() { + d := BackupDetails{Status: "success", Size: "4.2GB"} + data, err := json.Marshal(d) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("Backup/v1")) + Expect(m["status"]).To(Equal("success")) + Expect(m["size"]).To(Equal("4.2GB")) + }) + + It("should inject kind into empty struct", func() { + d := RollbackDetails{} + data, err := json.Marshal(d) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("Rollback/v1")) + Expect(m).To(HaveLen(1)) + }) + + It("should inject kind into ScalingDetails with numeric fields", func() { + d := ScalingDetails{FromReplicas: 2, ToReplicas: 5, ResourceType: "Deployment"} + data, err := json.Marshal(d) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("Scaling/v1")) + Expect(m["from_replicas"]).To(BeNumerically("==", 2)) + Expect(m["to_replicas"]).To(BeNumerically("==", 5)) + }) + + It("should inject kind into CostChangeDetails with float fields", func() { + d := CostChangeDetails{PreviousCost: 100.50, NewCost: 125.75, Currency: "USD"} + data, err := json.Marshal(d) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("CostChange/v1")) + Expect(m["previous_cost"]).To(BeNumerically("~", 100.50, 0.01)) + Expect(m["new_cost"]).To(BeNumerically("~", 125.75, 0.01)) + }) + + DescribeTable("all detail types implement ConfigChangeDetail", + func(d ConfigChangeDetail, expectedKind string) { + Expect(d.Kind()).To(Equal(expectedKind)) + + data, err := json.Marshal(d) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal(expectedKind)) + }, + Entry("UserChange", UserChangeDetails{UserName: "alice"}, "UserChange/v1"), + Entry("Screenshot", ScreenshotDetails{URL: "https://example.com"}, "Screenshot/v1"), + Entry("PermissionChange", PermissionChangeDetails{RoleName: "admin"}, "PermissionChange/v1"), + Entry("Deployment", DeploymentDetails{NewImage: "v2"}, "Deployment/v1"), + Entry("Promotion", PromotionDetails{ToEnvironment: "prod"}, "Promotion/v1"), + Entry("Approval", ApprovalDetails{ApprovedBy: "alice"}, "Approval/v1"), + Entry("Rollback", RollbackDetails{ToVersion: "v1"}, "Rollback/v1"), + Entry("Backup", BackupDetails{Status: "success"}, "Backup/v1"), + Entry("PlaybookExecution", PlaybookExecutionDetails{PlaybookName: "restart"}, "PlaybookExecution/v1"), + Entry("Scaling", ScalingDetails{ToReplicas: 3}, "Scaling/v1"), + Entry("Certificate", CertificateDetails{Subject: "*.example.com"}, "Certificate/v1"), + Entry("CostChange", CostChangeDetails{NewCost: 50}, "CostChange/v1"), + Entry("PipelineRun", PipelineRunDetails{Branch: "main"}, "PipelineRun/v1"), + ) +}) From 84e6d6bd9d646b6a2ef057fca7a00aeb53e9bf5c Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 7 Apr 2026 14:22:43 +0300 Subject: [PATCH 03/12] chore: ignore ginkgo config files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f5bf7b1a9..70b99bdd7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ out.html specs/*.md generate-openapi .todos/ +ginkgo*.json From 969c7bd79a7256869cef640e285124e76bd98e34 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Tue, 7 Apr 2026 14:23:49 +0300 Subject: [PATCH 04/12] refactor(query): extract parent lookup logic and support expanded resource selectors Extract repeated parent node lookup into findNearestAncestor helper. Add support for semicolon-delimited searches in resource selectors to enable union queries. Include Pretty field in query log entries for formatted output. --- query/config_tree.go | 39 ++++++++++++++++++-------------------- query/query_logger.go | 2 ++ query/resource_selector.go | 32 +++++++++++++++++-------------- types/resource_selector.go | 23 ++++++++++++++++++++++ 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/query/config_tree.go b/query/config_tree.go index ee9b2c854..bd11e1d45 100644 --- a/query/config_tree.go +++ b/query/config_tree.go @@ -183,16 +183,9 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi continue } if c.Path != "" { - segments := strings.Split(c.Path, ".") - if len(segments) >= 2 { - if parentStr := segments[len(segments)-2]; parentStr != "" { - if pid, err := uuid.Parse(parentStr); err == nil { - if parent, ok := nodes[pid]; ok { - parent.children = append(parent.children, nodes[c.ID]) - continue - } - } - } + if parent := findNearestAncestor(c.Path, nodes); parent != nil { + parent.children = append(parent.children, nodes[c.ID]) + continue } } targetNode.children = append(targetNode.children, nodes[c.ID]) @@ -224,17 +217,9 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi wired[rc.ID] = true node := nodes[rc.ID] if rc.Path != "" { - segments := strings.Split(rc.Path, ".") - // Last segment is the node's own ID (SetParent appends ci.ID), so use penultimate - if len(segments) >= 2 { - if parentStr := segments[len(segments)-2]; parentStr != "" { - if pid, err := uuid.Parse(parentStr); err == nil { - if parent, ok := nodes[pid]; ok && parent != node && !parentIDs[pid] { - parent.children = append(parent.children, node) - continue - } - } - } + if parent := findNearestAncestor(rc.Path, nodes); parent != nil && parent != node && !parentIDs[parent.ID] { + parent.children = append(parent.children, node) + continue } } targetNode.children = append(targetNode.children, node) @@ -250,6 +235,18 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi return toConfigTreeNode(root, make(map[*ptrNode]bool)) } +func findNearestAncestor(path string, nodes map[uuid.UUID]*ptrNode) *ptrNode { + segments := strings.Split(path, ".") + for i := len(segments) - 1; i >= 0; i-- { + if pid, err := uuid.Parse(segments[i]); err == nil { + if parent, ok := nodes[pid]; ok { + return parent + } + } + } + return nil +} + func toConfigTreeNode(n *ptrNode, visited map[*ptrNode]bool) *ConfigTreeNode { result := &ConfigTreeNode{ ConfigItem: n.ConfigItem, diff --git a/query/query_logger.go b/query/query_logger.go index 26cd693b6..0eb3d95d3 100644 --- a/query/query_logger.go +++ b/query/query_logger.go @@ -20,6 +20,7 @@ type QueryLogEntry struct { Duration int64 `json:"duration"` Error string `json:"error,omitempty"` Summary string `json:"summary,omitempty"` + Pretty string `json:"pretty"` } type QueryLog struct { @@ -148,6 +149,7 @@ func (t *QueryTimer) End(err *error) { label = label.AddText(fmt.Sprintf(" in %dms", elapsed.Milliseconds()), "text-gray-400") if t.queryLog != nil { + entry.Pretty = label.String() t.queryLog.Append(entry) } diff --git a/query/resource_selector.go b/query/resource_selector.go index cc1146d29..36ff0af4e 100644 --- a/query/resource_selector.go +++ b/query/resource_selector.go @@ -691,14 +691,16 @@ func queryTableWithResourceSelectors( var output []uuid.UUID for _, resourceSelector := range resourceSelectors { - items, err := queryResourceSelector[uuid.UUID](ctx, limit, []string{"id"}, resourceSelector, table) - if err != nil { - return nil, err - } + for _, expanded := range resourceSelector.Expand() { + items, err := queryResourceSelector[uuid.UUID](ctx, limit, []string{"id"}, expanded, table) + if err != nil { + return nil, err + } - output = append(output, items...) - if limit > 0 && len(output) >= limit { - return output[:limit], nil + output = append(output, items...) + if limit > 0 && len(output) >= limit { + return output[:limit], nil + } } } @@ -716,14 +718,16 @@ func QueryTableColumnsWithResourceSelectors[T any]( var output []T for _, resourceSelector := range resourceSelectors { - items, err := queryResourceSelector[T](ctx, limit, selectColumns, resourceSelector, table, clauses...) - if err != nil { - return nil, err - } + for _, expanded := range resourceSelector.Expand() { + items, err := queryResourceSelector[T](ctx, limit, selectColumns, expanded, table, clauses...) + if err != nil { + return nil, err + } - output = append(output, items...) - if limit > 0 && len(output) >= limit { - return output[:limit], nil + output = append(output, items...) + if limit > 0 && len(output) >= limit { + return output[:limit], nil + } } } diff --git a/types/resource_selector.go b/types/resource_selector.go index 62ec890e2..602f6b264 100644 --- a/types/resource_selector.go +++ b/types/resource_selector.go @@ -86,6 +86,29 @@ type ResourceSelector struct { Statuses Items `yaml:"statuses,omitempty" json:"statuses,omitempty"` } +// Expand splits a semicolon-delimited Search into multiple ResourceSelectors, +// each representing an independent query whose results are unioned. +func (rs ResourceSelector) Expand() []ResourceSelector { + if !strings.Contains(rs.Search, ";") { + return []ResourceSelector{rs} + } + segments := strings.Split(rs.Search, ";") + var result []ResourceSelector + for _, seg := range segments { + seg = strings.TrimSpace(seg) + if seg == "" { + continue + } + expanded := rs + expanded.Search = seg + result = append(result, expanded) + } + if len(result) == 0 { + return []ResourceSelector{rs} + } + return result +} + // ParseFilteringQuery parses a filtering query string. // It returns four slices: 'in', 'notIN', 'prefix', and 'suffix'. func ParseFilteringQuery(query string, decodeURL bool) (in []interface{}, notIN []interface{}, prefix, suffix []string, err error) { From 9cf5e7aed508cfd512afe731d8d94a8215c6e3a3 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 18:00:33 +0300 Subject: [PATCH 05/12] feat(types): extend change details with deployment, approval and source types --- types/config_changes.go | 116 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/types/config_changes.go b/types/config_changes.go index d47f3aded..4c40d0877 100644 --- a/types/config_changes.go +++ b/types/config_changes.go @@ -131,12 +131,118 @@ func (d PermissionChangeDetails) MarshalJSON() ([]byte, error) { return marshalWithKind(d.Kind(), raw(d)) } +type DeploymentType string + +const ( + ImageUpgrade DeploymentType = "ImageUpgrade" + ImageDowngrade DeploymentType = "ImageDowngrade" + Rollout DeploymentType = "Rollout" + Restart DeploymentType = "Restart" + Rollback DeploymentType = "Rollback" + ConfigurationChange DeploymentType = "ConfigurationChange" + ScaleUp DeploymentType = "ScaleUp" + ScaleDown DeploymentType = "ScaleDown" + ScaleIn DeploymentType = "ScaleIn" + ScaleOut DeploymentType = "ScaleOut" + PolicyChange DeploymentType = "PolicyChange" + SchemaChange DeploymentType = "SchemaChange" + DataMigration DeploymentType = "DataMigration" + DataFix DeploymentType = "DataFix" + OtherDeploymentChange DeploymentType = "Other" +) + +type ApproverType string + +const ( + ApproverTypeUser ApproverType = "User" + ApproverTypeGroup ApproverType = "Group" + ApproverTypeRole ApproverType = "Role" + ApproverTypeCI ApproverType = "CI" + ApproverTypeAuto ApproverType = "Auto" + ApproverTypeScan ApproverType = "Scan" + ApproverTypeTest ApproverType = "Test" + ApproverTypeCanary ApproverType = "Canary" +) + +type ApprovalStage string + +const ( + ApprovalStagePreDeployment ApprovalStage = "PreDeployment" + ApprovalStagePostDeployment ApprovalStage = "PostDeployment" + ApprovalStagePrePromotion ApprovalStage = "PrePromotion" + ApprovalStagePostPromotion ApprovalStage = "PostPromotion" + ApprovalStageManual ApprovalStage = "Manual" + ApprovalStageAutomated ApprovalStage = "Automated" +) + +type ApprovalStatus string + +const ( + ApprovalStatusApproved ApprovalStatus = "Approved" + ApprovalStatusRejected ApprovalStatus = "Rejected" + ApprovalStatusPending ApprovalStatus = "Pending" + ApprovalStatusExpired ApprovalStatus = "Expired" +) + +type Identity struct { + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` +} + +func (i Identity) IsEmpty() bool { + return i.ID == "" && i.Type == "" && i.Name == "" +} + +func (i Identity) Kind() string { return "Identity/v1" } +func (i Identity) MarshalJSON() ([]byte, error) { + type raw Identity + return marshalWithKind(i.Kind(), raw(i)) +} + +type Approval struct { + SubmittedBy Identity `json:"submitted_by,omitempty"` + Approver Identity `json:"approver,omitempty"` + ApproverType ApproverType `json:"approver_type,omitempty"` + Stage ApprovalStage `json:"stage,omitempty"` + Status ApprovalStatus `json:"status,omitempty"` + Message string `json:"message,omitempty"` +} + +type SourceType string + +const ( + SourceTypeGit SourceType = "Git" + SourceTypeHelm SourceType = "Helm" + SourceTypeDatabase SourceType = "Database" + SourceTypeOther SourceType = "Other" +) + +type GitSource struct { + URL string `json:"url,omitempty"` + Branch string `json:"branch,omitempty"` + CommitSHA string `json:"commit_sha,omitempty"` + Version string `json:"version,omitempty"` + Tags string `json:"tags,omitempty"` +} + +type Source struct { + Type SourceType `json:"type,omitempty"` + Name string `json:"name,omitempty"` + // For gitsource + URL string `json:"url,omitempty"` + + ID string `json:"id,omitempty"` +} + type DeploymentDetails struct { - PreviousImage string `json:"previous_image,omitempty"` - NewImage string `json:"new_image,omitempty"` - Container string `json:"container,omitempty"` - Namespace string `json:"namespace,omitempty"` - Strategy string `json:"strategy,omitempty"` + Type DeploymentType `json:"type,omitempty"` + Approvals []Approval `json:"approvals,omitempty"` + PreviousImage string `json:"previous_image,omitempty"` + NewImage string `json:"new_image,omitempty"` + Container string `json:"container,omitempty"` + Namespace string `json:"namespace,omitempty"` + Strategy string `json:"strategy,omitempty"` } func (d DeploymentDetails) Kind() string { return "Deployment/v1" } From f60360744655a40039f8aab854e186bb5fd1345a Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 12:01:05 +0300 Subject: [PATCH 06/12] refactor(schema)!: restructure config change schema to kind-discriminated union Replace reflective schema generation with handwritten kind-discriminated union schema. Adds UnmarshalChangeDetails for runtime deserialization. Introduces new detail types (Approval, Source, Environment, Event, Test, Promotion, PipelineRun, Change, ConfigChange, Restore, Backup, Dimension, Scale, GroupMembership) and removes old details types (DeploymentDetails, PromotionDetails, ApprovalDetails, RollbackDetails, BackupDetails, PlaybookExecutionDetails, ScalingDetails, CertificateDetails, CostChangeDetails, PipelineRunDetails). BREAKING CHANGE: restructure config change schema to kind-discriminated union --- hack/generate-schemas/main.go | 38 +- schema/openapi/change-types.schema.json | 1508 ++++++++++++++++++++--- types/config_changes.go | 667 +++++++--- types/config_changes_test.go | 387 +++++- 4 files changed, 2242 insertions(+), 358 deletions(-) diff --git a/hack/generate-schemas/main.go b/hack/generate-schemas/main.go index 24629a3f9..e68faefda 100644 --- a/hack/generate-schemas/main.go +++ b/hack/generate-schemas/main.go @@ -1,6 +1,8 @@ package main import ( + "encoding/json" + "fmt" "os" "path" @@ -10,22 +12,52 @@ import ( "github.com/spf13/cobra" ) -var schemas = map[string]any{ +var generatedSchemas = map[string]any{ "resource_selector": &types.ResourceSelector{}, "resource_selectors": &[]types.ResourceSelector{}, - "change-types": &types.ConfigChangeDetailsSchema{}, +} + +// change-types is maintained by hand because the reflective schema generator +// does not model the kind-discriminated union shape correctly. +var handwrittenSchemas = map[string]string{ + "change-types": "change-types.handwritten.schema.json", +} + +func writeHandwrittenSchema(dst, src string) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("unable to read handwritten schema %s: %w", src, err) + } + + if !json.Valid(data) { + return fmt.Errorf("handwritten schema %s is not valid json", src) + } + + if err := os.WriteFile(dst, data, 0644); err != nil { + return fmt.Errorf("unable to write handwritten schema to %s: %w", dst, err) + } + + return nil } var generateSchema = &cobra.Command{ Use: "generate-schema", Run: func(cmd *cobra.Command, args []string) { - for file, obj := range schemas { + for file, obj := range generatedSchemas { p := path.Join("../../schema/openapi", file+".schema.json") if err := openapi.WriteSchemaToFile(p, obj); err != nil { logger.Fatalf("unable to save schema: %v", err) } logger.Infof("Saved OpenAPI schema to %s", p) } + + for file, src := range handwrittenSchemas { + dst := path.Join("../../schema/openapi", file+".schema.json") + if err := writeHandwrittenSchema(dst, src); err != nil { + logger.Fatalf("unable to save handwritten schema: %v", err) + } + logger.Infof("Saved OpenAPI schema to %s", dst) + } }, } diff --git a/schema/openapi/change-types.schema.json b/schema/openapi/change-types.schema.json index c06babeb0..7964dbc41 100644 --- a/schema/openapi/change-types.schema.json +++ b/schema/openapi/change-types.schema.json @@ -3,308 +3,1426 @@ "$id": "https://github.com/flanksource/duty/types/config-change-details-schema", "$ref": "#/$defs/ConfigChangeDetailsSchema", "$defs": { - "ApprovalDetails": { + "ConfigChangeDetailsSchema": { + "description": "Kind-discriminated schema for config change payloads supported by UnmarshalChangeDetails.", + "oneOf": [ + { "$ref": "#/$defs/UserChangeDetails" }, + { "$ref": "#/$defs/ScreenshotDetails" }, + { "$ref": "#/$defs/PermissionChangeDetails" }, + { "$ref": "#/$defs/GroupMembership" }, + { "$ref": "#/$defs/Identity" }, + { "$ref": "#/$defs/Approval" }, + { "$ref": "#/$defs/GitSource" }, + { "$ref": "#/$defs/HelmSource" }, + { "$ref": "#/$defs/ImageSource" }, + { "$ref": "#/$defs/DatabaseSource" }, + { "$ref": "#/$defs/Source" }, + { "$ref": "#/$defs/Environment" }, + { "$ref": "#/$defs/Event" }, + { "$ref": "#/$defs/Test" }, + { "$ref": "#/$defs/Promotion" }, + { "$ref": "#/$defs/PipelineRun" }, + { "$ref": "#/$defs/Change" }, + { "$ref": "#/$defs/ConfigChange" }, + { "$ref": "#/$defs/Restore" }, + { "$ref": "#/$defs/Backup" }, + { "$ref": "#/$defs/Dimension" }, + { "$ref": "#/$defs/Scale" } + ], + "examples": [ + { + "kind": "UserChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "user_email": "alice@example.com", + "user_type": "human", + "group_id": "group-platform", + "group_name": "platform-admins", + "tenant": "acme" + }, + { + "kind": "PermissionChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "role_id": "role-admin", + "role_name": "cluster-admin", + "role_type": "kubernetes", + "scope": "prod-cluster" + }, + { + "kind": "Promotion/v1", + "id": "evt-promo-1", + "timestamp": "2026-04-10T12:00:00Z", + "from": { + "kind": "Environment/v1", + "name": "staging", + "stage": "Staging" + }, + "to": { + "kind": "Environment/v1", + "name": "production", + "stage": "Production" + }, + "source": { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456" + }, + "path": "deploy/production" + }, + "version": "v1.2.3", + "approvals": [ + { + "kind": "Approval/v1", + "id": "approval-1", + "submitted_by": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "approver": { + "kind": "Identity/v1", + "id": "user-456", + "type": "User", + "name": "bob@example.com" + }, + "stage": "Manual", + "status": "Approved" + } + ], + "artifact": "api" + }, + { + "kind": "Backup/v1", + "id": "backup-42", + "timestamp": "2026-04-10T11:30:00Z", + "end": "2026-04-10T11:36:12Z", + "backup_type": "Snapshot", + "created_by": { + "kind": "Identity/v1", + "type": "System:Auto", + "name": "nightly-backup" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production" + }, + "status": "Completed", + "size": "4.2GB", + "delta": "275MB" + }, + { + "kind": "ConfigChange/v1", + "id": "evt-config-1", + "timestamp": "2026-04-10T12:05:00Z", + "author": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production", + "identifier": "prod-cluster-1" + }, + "source": { + "kind": "Source/v1", + "image": { + "kind": "ImageSource/v1", + "registry": "ghcr.io", + "image": "flanksource/duty", + "version": "v1.2.3", + "sha": "sha256:1234abcd" + } + }, + "changes": [ + { + "kind": "Change/v1", + "path": ".spec.template.spec.containers[0].image", + "from": { + "image": "ghcr.io/flanksource/duty:v1.2.2" + }, + "to": { + "image": "ghcr.io/flanksource/duty:v1.2.3" + }, + "type": "update" + }, + { + "kind": "Change/v1", + "path": ".spec.replicas", + "from": { + "desired": "2" + }, + "to": { + "desired": "3" + }, + "type": "update" + } + ] + } + ] + }, + "StringMap": { + "type": "object", + "description": "String key-value metadata map.", + "additionalProperties": { + "type": "string" + } + }, + "AnyMap": { + "type": "object", + "description": "Object with arbitrary JSON values.", + "additionalProperties": true + }, + "IdentityType": { + "type": "string", + "enum": [ + "User", + "Group", + "Role", + "System:CI", + "System:Auto", + "System:Scan", + "System:Test", + "System:Canary" + ] + }, + "ApprovalStage": { + "type": "string", + "enum": [ + "PreDeployment", + "PostDeployment", + "PrePromotion", + "PostPromotion", + "Manual", + "Automated" + ] + }, + "ApprovalStatus": { + "type": "string", + "enum": [ + "Approved", + "Rejected", + "Pending", + "Expired" + ] + }, + "EnvironmentType": { + "type": "string", + "enum": [ + "Kubernetes", + "Cloud", + "On-Premises", + "Other" + ] + }, + "EnvironmentStage": { + "type": "string", + "enum": [ + "Development", + "Staging", + "Production", + "UAT", + "QA" + ] + }, + "TestingType": { + "type": "string", + "enum": [ + "Unit", + "Integration", + "End-to-End", + "Performance", + "Security" + ] + }, + "TestingStatus": { + "type": "string", + "enum": [ + "Pending", + "Running", + "Passed", + "Failed", + "Skipped", + "Error" + ] + }, + "TestingResult": { + "type": "string", + "enum": [ + "Flaky", + "Failed", + "Passed" + ] + }, + "Status": { + "type": "string", + "enum": [ + "Pending", + "Running", + "Timeout", + "Completed", + "Failed", + "Approved", + "Rejected" + ] + }, + "BackupType": { + "type": "string", + "enum": [ + "Dump", + "Snapshot", + "StorageBackup", + "Offsite", + "OffAccount", + "OffRegion" + ] + }, + "ScalingDimension": { + "type": "string", + "enum": [ + "CPU", + "Memory", + "Replicas", + "Custom" + ] + }, + "Identity": { + "type": "object", + "description": "Versioned identity payload for people, groups, roles, or system actors.", + "required": [ + "kind" + ], "properties": { - "playbook_id": { - "type": "string" + "kind": { + "const": "Identity/v1" }, - "run_id": { + "id": { "type": "string" }, - "approved_by": { - "type": "string" + "type": { + "$ref": "#/$defs/IdentityType" }, - "rejected_by": { - "type": "string" + "name": { + "type": "string", + "description": "Optional human-readable name for the identity, e.g. user name or group name." }, - "reason": { - "type": "string" + "comment": { + "type": "string", + "description": "Optional comment about the identity, e.g. reason for approval or details about the change." } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + { + "kind": "Identity/v1", + "id": "system-ci", + "type": "System:CI", + "name": "github-actions", + "comment": "Triggered by merge to main" + } + ] }, - "BackupDetails": { + "GitSource": { + "type": "object", + "description": "Git-backed source metadata.", + "required": [ + "kind" + ], "properties": { - "status": { - "type": "string" + "kind": { + "const": "GitSource/v1" }, - "size": { - "type": "string" + "url": { + "type": "string", + "format": "uri" }, - "duration": { + "branch": { "type": "string" }, - "backup_type": { + "commit_sha": { "type": "string" }, - "target": { + "version": { "type": "string" }, - "snapshot_id": { + "tags": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456", + "version": "v1.2.3", + "tags": "release,v1.2.3" + }, + { + "kind": "GitSource/v1", + "url": "ssh://git@github.com/flanksource/duty.git", + "branch": "release-1.2", + "commit_sha": "fedcba654321" + } + ] }, - "CertificateDetails": { + "HelmSource": { + "type": "object", + "description": "Helm chart source metadata.", + "required": [ + "kind" + ], "properties": { - "subject": { + "kind": { + "const": "HelmSource/v1" + }, + "chart_name": { "type": "string" }, - "issuer": { + "chart_version": { "type": "string" }, - "not_before": { + "repo_url": { + "type": "string", + "format": "uri" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "HelmSource/v1", + "chart_name": "ingress-nginx", + "chart_version": "4.10.0", + "repo_url": "https://kubernetes.github.io/ingress-nginx" + }, + { + "kind": "HelmSource/v1", + "chart_name": "duty", + "chart_version": "1.2.3", + "repo_url": "https://charts.example.com/platform" + } + ] + }, + "ImageSource": { + "type": "object", + "description": "Container image source metadata.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "ImageSource/v1" + }, + "registry": { "type": "string" }, - "not_after": { + "image": { "type": "string" }, - "serial": { + "version": { "type": "string" }, - "dns_names": { + "sha": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "ImageSource/v1", + "registry": "ghcr.io", + "image": "flanksource/duty", + "version": "v1.2.3", + "sha": "sha256:1234abcd" + }, + { + "kind": "ImageSource/v1", + "registry": "123456789012.dkr.ecr.us-east-1.amazonaws.com", + "image": "payments/api", + "version": "2026.04.10-1" + } + ] }, - "ConfigChangeDetailsSchema": { + "DatabaseSource": { + "type": "object", + "description": "Database source metadata.", + "required": [ + "kind" + ], "properties": { - "UserChange/v1": { - "$ref": "#/$defs/UserChangeDetails" + "kind": { + "const": "DatabaseSource/v1" + }, + "type": { + "type": "string", + "description": "Database type, e.g. PostgreSQL, MySQL, MongoDB." }, - "Screenshot/v1": { - "$ref": "#/$defs/ScreenshotDetails" + "name": { + "type": "string", + "description": "Database name." }, - "PermissionChange/v1": { - "$ref": "#/$defs/PermissionChangeDetails" + "schema": { + "type": "string", + "description": "Schema name, e.g. public." }, - "Deployment/v1": { - "$ref": "#/$defs/DeploymentDetails" + "version": { + "type": "string", + "description": "Database version." }, - "Promotion/v1": { - "$ref": "#/$defs/PromotionDetails" + "endpoint": { + "type": "string", + "description": "Server or cluster endpoint." + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "DatabaseSource/v1", + "type": "PostgreSQL", + "name": "appdb", + "schema": "public", + "version": "16", + "endpoint": "appdb.cluster-123.us-east-1.rds.amazonaws.com:5432" + }, + { + "kind": "DatabaseSource/v1", + "type": "MongoDB", + "name": "audit", + "version": "7.0", + "endpoint": "mongo.example.internal:27017" + } + ] + }, + "Source": { + "type": "object", + "description": "Versioned source envelope for Git, Helm, image, or database change provenance.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Source/v1" }, - "Approval/v1": { - "$ref": "#/$defs/ApprovalDetails" + "git": { + "$ref": "#/$defs/GitSource" }, - "Rollback/v1": { - "$ref": "#/$defs/RollbackDetails" + "helm": { + "$ref": "#/$defs/HelmSource" }, - "Backup/v1": { - "$ref": "#/$defs/BackupDetails" + "image": { + "$ref": "#/$defs/ImageSource" }, - "PlaybookExecution/v1": { - "$ref": "#/$defs/PlaybookExecutionDetails" + "database": { + "$ref": "#/$defs/DatabaseSource" }, - "Scaling/v1": { - "$ref": "#/$defs/ScalingDetails" + "kustomization": { + "$ref": "#/$defs/GitSource" }, - "Certificate/v1": { - "$ref": "#/$defs/CertificateDetails" + "argocd": { + "$ref": "#/$defs/GitSource" }, - "CostChange/v1": { - "$ref": "#/$defs/CostChangeDetails" + "other": { + "type": "string" }, - "PipelineRun/v1": { - "$ref": "#/$defs/PipelineRunDetails" + "path": { + "type": "string", + "description": "Optional path within the source, e.g. file path in git repo or chart path in a Helm repo." } }, "additionalProperties": false, - "type": "object", - "description": "ConfigChangeDetailsSchema is a union type used for JSON schema generation.\nIt is never instantiated at runtime." + "examples": [ + { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main" + }, + "path": "deploy/production" + }, + { + "kind": "Source/v1", + "helm": { + "kind": "HelmSource/v1", + "chart_name": "duty", + "chart_version": "1.2.3", + "repo_url": "https://charts.example.com/platform" + }, + "path": "charts/duty" + }, + { + "kind": "Source/v1", + "argocd": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/platform-config.git", + "branch": "main", + "commit_sha": "bbccddeeff00" + }, + "path": "clusters/prod/api" + } + ] }, - "CostChangeDetails": { + "Environment": { + "type": "object", + "description": "Versioned environment descriptor for change events.", + "required": [ + "kind" + ], "properties": { - "previous_cost": { - "type": "number" + "kind": { + "const": "Environment/v1" }, - "new_cost": { - "type": "number" - }, - "currency": { + "name": { "type": "string" }, - "period": { + "description": { "type": "string" }, - "reason": { + "type": { + "$ref": "#/$defs/EnvironmentType", + "description": "Optional type of environment, e.g. Kubernetes, Cloud, On-Premises." + }, + "stage": { + "$ref": "#/$defs/EnvironmentStage", + "description": "Optional stage or lifecycle phase of the environment." + }, + "identifier": { "type": "string" + }, + "tags": { + "$ref": "#/$defs/StringMap", + "description": "Optional tags for team, owner, namespace, cost center, or other metadata." } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production", + "identifier": "prod-cluster-1", + "tags": { + "cluster": "prod-cluster-1", + "namespace": "platform", + "owner": "platform-team" + } + }, + { + "kind": "Environment/v1", + "name": "staging", + "description": "Pre-production validation environment", + "type": "Cloud", + "stage": "Staging", + "identifier": "staging-account" + } + ] }, - "DeploymentDetails": { + "Event": { + "type": "object", + "description": "Common event metadata shared by versioned change payloads.", + "required": [ + "kind" + ], "properties": { - "previous_image": { - "type": "string" + "kind": { + "const": "Event/v1" }, - "new_image": { + "id": { "type": "string" }, - "container": { + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Event/v1", + "id": "evt-123", + "url": "https://github.com/flanksource/duty/actions/runs/123456789", + "tags": { + "service": "api", + "team": "platform" + }, + "properties": { + "cluster": "prod-cluster-1", + "namespace": "platform" + }, + "timestamp": "2026-04-10T12:00:00Z" + } + ] + }, + "Approval": { + "type": "object", + "description": "Approval event with optional submitter and approver identities.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Approval/v1" }, - "namespace": { + "id": { "type": "string" }, - "strategy": { + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" + }, + "submitted_by": { + "$ref": "#/$defs/Identity", + "description": "Optional identity of the person or system that submitted the approval request." + }, + "approver": { + "$ref": "#/$defs/Identity", + "description": "Optional identity of the person or system that approved or rejected the change." + }, + "stage": { + "$ref": "#/$defs/ApprovalStage" + }, + "status": { + "$ref": "#/$defs/ApprovalStatus" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Approval/v1", + "id": "evt-approval-1", + "timestamp": "2026-04-10T11:58:00Z", + "submitted_by": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "approver": { + "kind": "Identity/v1", + "id": "user-456", + "type": "User", + "name": "bob@example.com" + }, + "stage": "Manual", + "status": "Approved" + }, + { + "kind": "Approval/v1", + "id": "evt-approval-2", + "submitted_by": { + "kind": "Identity/v1", + "type": "System:CI", + "name": "github-actions" + }, + "stage": "Automated", + "status": "Pending" + } + ] }, - "PermissionChangeDetails": { + "Test": { + "type": "object", + "description": "Versioned test execution event.", + "required": [ + "kind" + ], "properties": { - "user_id": { - "type": "string" + "kind": { + "const": "Test/v1" }, - "user_name": { + "id": { "type": "string" }, - "group_id": { + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" }, - "group_name": { + "name": { "type": "string" }, - "role_id": { + "description": { "type": "string" }, - "role_name": { + "type": { + "$ref": "#/$defs/TestingType" + }, + "status": { + "$ref": "#/$defs/TestingStatus" + }, + "result": { + "$ref": "#/$defs/TestingResult" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Test/v1", + "id": "evt-test-1", + "timestamp": "2026-04-10T11:50:00Z", + "name": "smoke", + "description": "Validates the production-ready deployment path", + "type": "End-to-End", + "status": "Passed", + "result": "Passed" + }, + { + "kind": "Test/v1", + "id": "evt-test-2", + "name": "container-scan", + "type": "Security", + "status": "Failed", + "result": "Failed" + } + ] + }, + "Promotion": { + "type": "object", + "description": "Promotion event between environments.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Promotion/v1" + }, + "id": { "type": "string" }, - "role_type": { + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" }, - "scope": { + "from": { + "$ref": "#/$defs/Environment", + "description": "Optional source environment for the promotion." + }, + "to": { + "$ref": "#/$defs/Environment", + "description": "Optional target environment for the promotion." + }, + "source": { + "$ref": "#/$defs/Source", + "description": "Optional source for the promotion, e.g. Git repo, Helm chart, image, or database metadata." + }, + "version": { + "type": "string", + "description": "Optional version or identifier for the promoted artifact." + }, + "approvals": { + "type": "array", + "items": { + "$ref": "#/$defs/Approval" + }, + "description": "Optional list of approvals attached to the promotion." + }, + "artifact": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Promotion/v1", + "id": "evt-promo-1", + "timestamp": "2026-04-10T12:00:00Z", + "from": { + "kind": "Environment/v1", + "name": "staging", + "stage": "Staging" + }, + "to": { + "kind": "Environment/v1", + "name": "production", + "stage": "Production" + }, + "source": { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456" + }, + "path": "deploy/production" + }, + "version": "v1.2.3", + "approvals": [ + { + "kind": "Approval/v1", + "id": "approval-1", + "submitted_by": { + "kind": "Identity/v1", + "type": "User", + "name": "alice@example.com" + }, + "approver": { + "kind": "Identity/v1", + "type": "User", + "name": "bob@example.com" + }, + "stage": "Manual", + "status": "Approved" + } + ], + "artifact": "api" + } + ] }, - "PipelineRunDetails": { + "PipelineRun": { + "type": "object", + "description": "Pipeline run event scoped to an environment.", + "required": [ + "kind" + ], "properties": { - "pipeline_id": { - "type": "string" + "kind": { + "const": "PipelineRun/v1" }, - "pipeline_name": { + "id": { "type": "string" }, - "run_id": { - "type": "string" + "url": { + "type": "string", + "format": "uri" }, - "run_number": { - "type": "integer" + "tags": { + "$ref": "#/$defs/StringMap" }, - "branch": { + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" }, + "environment": { + "$ref": "#/$defs/Environment" + }, "status": { - "type": "string" + "$ref": "#/$defs/Status" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "PipelineRun/v1", + "id": "evt-pipeline-1", + "url": "https://github.com/flanksource/duty/actions/runs/123456789", + "timestamp": "2026-04-10T11:45:00Z", + "environment": { + "kind": "Environment/v1", + "name": "production", + "stage": "Production" + }, + "status": "Running" + }, + { + "kind": "PipelineRun/v1", + "id": "evt-pipeline-2", + "environment": { + "kind": "Environment/v1", + "name": "staging", + "stage": "Staging" + }, + "status": "Completed" + } + ] + }, + "Change": { + "type": "object", + "description": "Single field-level change within a larger config change payload.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Change/v1" }, - "duration": { + "path": { "type": "string" }, - "error": { + "from": { + "$ref": "#/$defs/AnyMap" + }, + "to": { + "$ref": "#/$defs/AnyMap" + }, + "type": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Change/v1", + "path": ".spec.replicas", + "from": { + "desired": "2" + }, + "to": { + "desired": "3" + }, + "type": "update" + }, + { + "kind": "Change/v1", + "path": ".metadata.annotations.release", + "to": { + "release": "2026.04.10" + }, + "type": "create" + } + ] }, - "PlaybookExecutionDetails": { + "ConfigChange": { + "type": "object", + "description": "Top-level config change payload with author, environment, source, and individual changes.", + "required": [ + "kind" + ], "properties": { - "playbook_id": { - "type": "string" + "kind": { + "const": "ConfigChange/v1" }, - "playbook_name": { + "id": { "type": "string" }, - "run_id": { - "type": "string" + "url": { + "type": "string", + "format": "uri" }, - "status": { + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" }, - "duration": { + "author": { + "$ref": "#/$defs/Identity" + }, + "changes": { + "type": "array", + "items": { + "$ref": "#/$defs/Change" + } + }, + "environment": { + "$ref": "#/$defs/Environment" + }, + "source": { + "$ref": "#/$defs/Source" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "ConfigChange/v1", + "id": "evt-config-1", + "url": "https://github.com/flanksource/duty/pull/123", + "timestamp": "2026-04-10T12:05:00Z", + "author": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production", + "identifier": "prod-cluster-1" + }, + "source": { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456" + }, + "path": "deploy/production" + }, + "changes": [ + { + "kind": "Change/v1", + "path": ".spec.template.spec.containers[0].image", + "from": { + "image": "ghcr.io/flanksource/duty:v1.2.2" + }, + "to": { + "image": "ghcr.io/flanksource/duty:v1.2.3" + }, + "type": "update" + }, + { + "kind": "Change/v1", + "path": ".spec.replicas", + "from": { + "desired": "2" + }, + "to": { + "desired": "3" + }, + "type": "update" + } + ] + } + ] + }, + "Restore": { + "type": "object", + "description": "Restore event between environments.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Restore/v1" + }, + "id": { "type": "string" }, - "error": { + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" + }, + "from": { + "$ref": "#/$defs/Environment", + "description": "Optional source environment for the restore." + }, + "to": { + "$ref": "#/$defs/Environment", + "description": "Optional target environment for the restore." + }, + "source": { + "$ref": "#/$defs/Source", + "description": "Optional source metadata for the restore." + }, + "status": { + "$ref": "#/$defs/Status", + "description": "Optional status of the restore." } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Restore/v1", + "id": "evt-restore-1", + "timestamp": "2026-04-10T13:10:00Z", + "from": { + "kind": "Environment/v1", + "name": "backup-storage", + "type": "Cloud", + "stage": "Production" + }, + "to": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production" + }, + "source": { + "kind": "Source/v1", + "database": { + "kind": "DatabaseSource/v1", + "type": "PostgreSQL", + "name": "appdb", + "schema": "public", + "version": "16" + } + }, + "status": "Completed" + } + ] }, - "PromotionDetails": { + "Backup": { + "type": "object", + "description": "Backup event with actor, environment, event metadata, and backup result fields.", + "required": [ + "kind" + ], "properties": { - "from_environment": { + "kind": { + "const": "Backup/v1" + }, + "backup_type": { + "$ref": "#/$defs/BackupType" + }, + "created_by": { + "$ref": "#/$defs/Identity" + }, + "environment": { + "$ref": "#/$defs/Environment" + }, + "id": { "type": "string" }, - "to_environment": { + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { "type": "string" }, - "version": { + "end": { "type": "string" }, - "artifact": { + "status": { + "$ref": "#/$defs/Status" + }, + "size": { + "type": "string" + }, + "delta": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Backup/v1", + "id": "backup-42", + "backup_type": "Snapshot", + "created_by": { + "kind": "Identity/v1", + "type": "System:Auto", + "name": "nightly-backup" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production" + }, + "timestamp": "2026-04-10T11:30:00Z", + "end": "2026-04-10T11:36:12Z", + "status": "Completed", + "size": "4.2GB", + "delta": "275MB" + } + ] }, - "RollbackDetails": { + "Dimension": { + "type": "object", + "description": "Dimension values used by scale events.", + "required": [ + "kind" + ], "properties": { - "from_version": { - "type": "string" + "kind": { + "const": "Dimension/v1" }, - "to_version": { + "min": { "type": "string" }, - "reason": { + "max": { "type": "string" }, - "trigger": { + "desired": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Dimension/v1", + "min": "2", + "max": "10", + "desired": "3" + } + ] }, - "ScalingDetails": { + "Scale": { + "type": "object", + "description": "Scale event payload with before and after dimension values.", + "required": [ + "kind" + ], "properties": { - "from_replicas": { - "type": "integer" + "kind": { + "const": "Scale/v1" }, - "to_replicas": { - "type": "integer" + "dimension": { + "$ref": "#/$defs/ScalingDimension" + }, + "previous_value": { + "$ref": "#/$defs/Dimension" + }, + "value": { + "$ref": "#/$defs/Dimension" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Scale/v1", + "dimension": "Replicas", + "previous_value": { + "kind": "Dimension/v1", + "min": "2", + "max": "10", + "desired": "2" + }, + "value": { + "kind": "Dimension/v1", + "min": "2", + "max": "10", + "desired": "5" + } + }, + { + "kind": "Scale/v1", + "dimension": "CPU", + "previous_value": { + "kind": "Dimension/v1", + "desired": "500m" + }, + "value": { + "kind": "Dimension/v1", + "desired": "1500m" + } + } + ] + }, + "UserChangeDetails": { + "type": "object", + "description": "User change payload.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "UserChange/v1" + }, + "user_id": { + "type": "string" }, - "resource_type": { + "user_name": { + "type": "string" + }, + "user_email": { + "type": "string" + }, + "user_type": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "group_name": { "type": "string" }, - "trigger": { + "tenant": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "UserChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "user_email": "alice@example.com", + "user_type": "human", + "group_id": "group-platform", + "group_name": "platform-admins" + }, + { + "kind": "UserChange/v1", + "user_id": "svc-789", + "user_name": "deploy-bot", + "user_type": "service-account", + "tenant": "acme" + } + ] }, "ScreenshotDetails": { + "type": "object", + "description": "Screenshot artifact metadata.", + "required": [ + "kind" + ], "properties": { + "kind": { + "const": "Screenshot/v1" + }, "artifact_id": { "type": "string" }, "url": { - "type": "string" + "type": "string", + "format": "uri" }, "content_type": { "type": "string" @@ -317,34 +1435,144 @@ } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "Screenshot/v1", + "artifact_id": "artifact-123", + "url": "https://example.com/screenshot.png", + "content_type": "image/png", + "width": 1920, + "height": 1080 + }, + { + "kind": "Screenshot/v1", + "artifact_id": "artifact-456", + "url": "https://example.com/error-state.jpg", + "content_type": "image/jpeg", + "width": 1280, + "height": 720 + } + ] }, - "UserChangeDetails": { + "PermissionChangeDetails": { + "type": "object", + "description": "Permission or role assignment change payload.", + "required": [ + "kind" + ], "properties": { + "kind": { + "const": "PermissionChange/v1" + }, "user_id": { "type": "string" }, "user_name": { "type": "string" }, - "user_email": { + "group_id": { "type": "string" }, - "user_type": { + "group_name": { "type": "string" }, - "group_id": { + "role_id": { "type": "string" }, - "group_name": { + "role_name": { + "type": "string" + }, + "role_type": { + "type": "string" + }, + "scope": { "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "PermissionChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "group_id": "group-platform", + "group_name": "platform-admins", + "role_id": "role-admin", + "role_name": "admin", + "role_type": "kubernetes", + "scope": "cluster-1" + }, + { + "kind": "PermissionChange/v1", + "group_id": "group-observability", + "group_name": "observability-team", + "role_id": "role-viewer", + "role_name": "viewer", + "role_type": "rbac", + "scope": "grafana-prod" + } + ] + }, + "GroupMembership": { + "type": "object", + "description": "Group membership change payload (member added to or removed from a group).", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "GroupMembership/v1" + }, + "group": { + "$ref": "#/$defs/Identity" + }, + "member": { + "$ref": "#/$defs/Identity" + }, + "action": { + "type": "string", + "enum": ["Added", "Removed"] }, "tenant": { "type": "string" } }, "additionalProperties": false, - "type": "object" + "examples": [ + { + "kind": "GroupMembership/v1", + "group": { + "kind": "Identity/v1", + "id": "group-platform", + "name": "platform-admins", + "type": "Group" + }, + "member": { + "kind": "Identity/v1", + "id": "user-123", + "name": "alice@example.com", + "type": "User" + }, + "action": "Added", + "tenant": "acme" + }, + { + "kind": "GroupMembership/v1", + "group": { + "kind": "Identity/v1", + "id": "group-observability", + "name": "observability-team", + "type": "Group" + }, + "member": { + "kind": "Identity/v1", + "id": "svc-789", + "name": "deploy-bot", + "type": "User" + }, + "action": "Removed" + } + ] } } -} \ No newline at end of file +} diff --git a/types/config_changes.go b/types/config_changes.go index 4c40d0877..49641de82 100644 --- a/types/config_changes.go +++ b/types/config_changes.go @@ -1,6 +1,11 @@ package types -import "encoding/json" +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" +) // ConfigChangeDetail is implemented by all typed detail structs // for the ConfigChange.Details field. @@ -8,6 +13,31 @@ type ConfigChangeDetail interface { Kind() string } +var configChangeDetailTypes = []ConfigChangeDetail{ + UserChangeDetails{}, + ScreenshotDetails{}, + PermissionChangeDetails{}, + GroupMembership{}, + Identity{}, + Approval{}, + GitSource{}, + HelmSource{}, + ImageSource{}, + DatabaseSource{}, + Source{}, + Environment{}, + Event{}, + Test{}, + Promotion{}, + PipelineRun{}, + Change{}, + ConfigChange{}, + Restore{}, + Backup{}, + Dimension{}, + Scale{}, +} + // Change type constants. const ( ChangeTypeCreate = "CREATE" @@ -66,22 +96,46 @@ func marshalWithKind(kind string, v any) ([]byte, error) { return []byte(`{"kind":"` + kind + `"}`), nil } -// ConfigChangeDetailsSchema is a union type used for JSON schema generation. -// It is never instantiated at runtime. -type ConfigChangeDetailsSchema struct { - UserChange *UserChangeDetails `json:"UserChange/v1,omitempty"` - Screenshot *ScreenshotDetails `json:"Screenshot/v1,omitempty"` - PermissionChange *PermissionChangeDetails `json:"PermissionChange/v1,omitempty"` - Deployment *DeploymentDetails `json:"Deployment/v1,omitempty"` - Promotion *PromotionDetails `json:"Promotion/v1,omitempty"` - Approval *ApprovalDetails `json:"Approval/v1,omitempty"` - Rollback *RollbackDetails `json:"Rollback/v1,omitempty"` - Backup *BackupDetails `json:"Backup/v1,omitempty"` - PlaybookExecution *PlaybookExecutionDetails `json:"PlaybookExecution/v1,omitempty"` - Scaling *ScalingDetails `json:"Scaling/v1,omitempty"` - Certificate *CertificateDetails `json:"Certificate/v1,omitempty"` - CostChange *CostChangeDetails `json:"CostChange/v1,omitempty"` - PipelineRun *PipelineRunDetails `json:"PipelineRun/v1,omitempty"` +func pointerIfNotZero[T any](v T) *T { + if reflect.ValueOf(v).IsZero() { + return nil + } + return &v +} + +// UnmarshalChangeDetails unmarshals the details field of a ConfigChange into the appropriate typed struct based on the "kind" field. +func UnmarshalChangeDetails(data []byte) (ConfigChangeDetail, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nil, nil + } + + var envelope struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(trimmed, &envelope); err != nil { + return nil, fmt.Errorf("decode config change details envelope: %w", err) + } + + for _, candidate := range configChangeDetailTypes { + if candidate.Kind() != envelope.Kind { + continue + } + + value := reflect.New(reflect.TypeOf(candidate)) + if err := json.Unmarshal(trimmed, value.Interface()); err != nil { + return nil, err + } + + detail, ok := value.Elem().Interface().(ConfigChangeDetail) + if !ok { + return nil, fmt.Errorf("decoded config change detail %q does not implement ConfigChangeDetail", envelope.Kind) + } + + return detail, nil + } + + return nil, fmt.Errorf("unknown config change detail kind %q", envelope.Kind) } type UserChangeDetails struct { @@ -131,6 +185,36 @@ func (d PermissionChangeDetails) MarshalJSON() ([]byte, error) { return marshalWithKind(d.Kind(), raw(d)) } +type GroupMembershipAction string + +const ( + GroupMembershipActionAdded GroupMembershipAction = "Added" + GroupMembershipActionRemoved GroupMembershipAction = "Removed" +) + +type GroupMembership struct { + Group Identity `json:"group,omitempty"` + Member Identity `json:"member,omitempty"` + Action GroupMembershipAction `json:"action,omitempty"` + Tenant string `json:"tenant,omitempty"` +} + +func (d GroupMembership) Kind() string { return "GroupMembership/v1" } +func (d GroupMembership) MarshalJSON() ([]byte, error) { + type payload struct { + Group *Identity `json:"group,omitempty"` + Member *Identity `json:"member,omitempty"` + Action GroupMembershipAction `json:"action,omitempty"` + Tenant string `json:"tenant,omitempty"` + } + return marshalWithKind(d.Kind(), payload{ + Group: pointerIfNotZero(d.Group), + Member: pointerIfNotZero(d.Member), + Action: d.Action, + Tenant: d.Tenant, + }) +} + type DeploymentType string const ( @@ -151,17 +235,17 @@ const ( OtherDeploymentChange DeploymentType = "Other" ) -type ApproverType string +type IdentityType string const ( - ApproverTypeUser ApproverType = "User" - ApproverTypeGroup ApproverType = "Group" - ApproverTypeRole ApproverType = "Role" - ApproverTypeCI ApproverType = "CI" - ApproverTypeAuto ApproverType = "Auto" - ApproverTypeScan ApproverType = "Scan" - ApproverTypeTest ApproverType = "Test" - ApproverTypeCanary ApproverType = "Canary" + IdentityTypeUser IdentityType = "User" + IdentityTypeGroup IdentityType = "Group" + IdentityTypeRole IdentityType = "Role" + IdentityTypeCI IdentityType = "System:CI" + IdentityTypeAuto IdentityType = "System:Auto" + IdentityTypeScan IdentityType = "System:Scan" + IdentityTypeTest IdentityType = "System:Test" + IdentityTypeCanary IdentityType = "System:Canary" ) type ApprovalStage string @@ -185,9 +269,12 @@ const ( ) type Identity struct { - ID string `json:"id,omitempty"` - Type string `json:"type,omitempty"` + ID string `json:"id,omitempty"` + Type IdentityType `json:"type,omitempty"` + // Optional human-readable name for the identity, e.g. user name or group name. Not required if ID is present and meaningful on its own. Name string `json:"name,omitempty"` + // Optional comment about the identity, e.g. reason for approval/rejection, or details about the change. + Comment string `json:"comment,omitempty"` } func (i Identity) IsEmpty() bool { @@ -201,23 +288,34 @@ func (i Identity) MarshalJSON() ([]byte, error) { } type Approval struct { - SubmittedBy Identity `json:"submitted_by,omitempty"` - Approver Identity `json:"approver,omitempty"` - ApproverType ApproverType `json:"approver_type,omitempty"` - Stage ApprovalStage `json:"stage,omitempty"` - Status ApprovalStatus `json:"status,omitempty"` - Message string `json:"message,omitempty"` + Event `json:",inline"` + // Optional identity of the person or system that submitted the approval request. May be empty for automated approvals or when the submitter is unknown. + SubmittedBy *Identity `json:"submitted_by,omitempty"` + // Optional identity of the person or system that approved or rejected the change. May be empty if the approval is still pending or if the approver is unknown. + Approver *Identity `json:"approver,omitempty"` + Stage ApprovalStage `json:"stage,omitempty"` + Status ApprovalStatus `json:"status,omitempty"` +} + +func (a Approval) Kind() string { return "Approval/v1" } +func (a Approval) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + SubmittedBy *Identity `json:"submitted_by,omitempty"` + Approver *Identity `json:"approver,omitempty"` + Stage ApprovalStage `json:"stage,omitempty"` + Status ApprovalStatus `json:"status,omitempty"` + } + return marshalWithKind(a.Kind(), payload{ + rawEvent: rawEvent(a.Event), + SubmittedBy: a.SubmittedBy, + Approver: a.Approver, + Stage: a.Stage, + Status: a.Status, + }) } -type SourceType string - -const ( - SourceTypeGit SourceType = "Git" - SourceTypeHelm SourceType = "Helm" - SourceTypeDatabase SourceType = "Database" - SourceTypeOther SourceType = "Other" -) - type GitSource struct { URL string `json:"url,omitempty"` Branch string `json:"branch,omitempty"` @@ -226,156 +324,409 @@ type GitSource struct { Tags string `json:"tags,omitempty"` } -type Source struct { - Type SourceType `json:"type,omitempty"` - Name string `json:"name,omitempty"` - // For gitsource - URL string `json:"url,omitempty"` - - ID string `json:"id,omitempty"` +func (s GitSource) Kind() string { return "GitSource/v1" } +func (s GitSource) MarshalJSON() ([]byte, error) { + type raw GitSource + return marshalWithKind(s.Kind(), raw(s)) } -type DeploymentDetails struct { - Type DeploymentType `json:"type,omitempty"` - Approvals []Approval `json:"approvals,omitempty"` - PreviousImage string `json:"previous_image,omitempty"` - NewImage string `json:"new_image,omitempty"` - Container string `json:"container,omitempty"` - Namespace string `json:"namespace,omitempty"` - Strategy string `json:"strategy,omitempty"` +type HelmSource struct { + ChartName string `json:"chart_name,omitempty"` + ChartVersion string `json:"chart_version,omitempty"` + RepoURL string `json:"repo_url,omitempty"` } -func (d DeploymentDetails) Kind() string { return "Deployment/v1" } -func (d DeploymentDetails) MarshalJSON() ([]byte, error) { - type raw DeploymentDetails - return marshalWithKind(d.Kind(), raw(d)) +func (s HelmSource) Kind() string { return "HelmSource/v1" } +func (s HelmSource) MarshalJSON() ([]byte, error) { + type raw HelmSource + return marshalWithKind(s.Kind(), raw(s)) } -type PromotionDetails struct { - FromEnvironment string `json:"from_environment,omitempty"` - ToEnvironment string `json:"to_environment,omitempty"` - Version string `json:"version,omitempty"` - Artifact string `json:"artifact,omitempty"` +type ImageSource struct { + Registry string `json:"registry,omitempty"` + ImageName string `json:"image,omitempty"` + Version string `json:"version,omitempty"` + SHA string `json:"sha,omitempty"` } -func (d PromotionDetails) Kind() string { return "Promotion/v1" } -func (d PromotionDetails) MarshalJSON() ([]byte, error) { - type raw PromotionDetails - return marshalWithKind(d.Kind(), raw(d)) +func (s ImageSource) Kind() string { return "ImageSource/v1" } +func (s ImageSource) MarshalJSON() ([]byte, error) { + type raw ImageSource + return marshalWithKind(s.Kind(), raw(s)) } -type ApprovalDetails struct { - PlaybookID string `json:"playbook_id,omitempty"` - RunID string `json:"run_id,omitempty"` - ApprovedBy string `json:"approved_by,omitempty"` - RejectedBy string `json:"rejected_by,omitempty"` - Reason string `json:"reason,omitempty"` +type DatabaseSource struct { + // Database type, e.g. "PostgreSQL", "MySQL", "MongoDB" + Type string `json:"type,omitempty"` + // Database name, e.g. "mydb" + Name string `json:"name,omitempty"` + // Schema name, e.g. "public" + SchemaName string `json:"schema,omitempty"` + // Database version, e.g. "12.3" + Version string `json:"version,omitempty"` + // Server or cluster endpoint, e.g. "mydb.cluster-123.us-east-1.rds.amazonaws.com:5432" + Endpoint string `json:"endpoint,omitempty"` } -func (d ApprovalDetails) Kind() string { return "Approval/v1" } -func (d ApprovalDetails) MarshalJSON() ([]byte, error) { - type raw ApprovalDetails - return marshalWithKind(d.Kind(), raw(d)) +func (s DatabaseSource) Kind() string { return "DatabaseSource/v1" } +func (s DatabaseSource) MarshalJSON() ([]byte, error) { + type raw DatabaseSource + return marshalWithKind(s.Kind(), raw(s)) } -type RollbackDetails struct { - FromVersion string `json:"from_version,omitempty"` - ToVersion string `json:"to_version,omitempty"` - Reason string `json:"reason,omitempty"` - Trigger string `json:"trigger,omitempty"` +type Source struct { + Git *GitSource `json:"git,omitempty"` + Helm *HelmSource `json:"helm,omitempty"` + Image *ImageSource `json:"image,omitempty"` + Database *DatabaseSource `json:"database,omitempty"` + KustomizationSource *GitSource `json:"kustomization,omitempty"` + ArgocdSource *GitSource `json:"argocd,omitempty"` + OtherSource *string `json:"other,omitempty"` + Path string `json:"path,omitempty"` // Optional path within the source, e.g. file path in git repo or chart path in Helm repo } -func (d RollbackDetails) Kind() string { return "Rollback/v1" } -func (d RollbackDetails) MarshalJSON() ([]byte, error) { - type raw RollbackDetails - return marshalWithKind(d.Kind(), raw(d)) +func (s Source) Kind() string { return "Source/v1" } +func (s Source) MarshalJSON() ([]byte, error) { + type raw Source + return marshalWithKind(s.Kind(), raw(s)) } -type BackupDetails struct { - Status string `json:"status,omitempty"` - Size string `json:"size,omitempty"` - Duration string `json:"duration,omitempty"` - BackupType string `json:"backup_type,omitempty"` - Target string `json:"target,omitempty"` - SnapshotID string `json:"snapshot_id,omitempty"` -} +type EnvironmentType string +type EnvironmentStage string -func (d BackupDetails) Kind() string { return "Backup/v1" } -func (d BackupDetails) MarshalJSON() ([]byte, error) { - type raw BackupDetails - return marshalWithKind(d.Kind(), raw(d)) -} +const ( + EnvironmentStageDevelopment EnvironmentStage = "Development" + EnvironmentStageStaging EnvironmentStage = "Staging" + EnvironmentStageProduction EnvironmentStage = "Production" + EnvironmentStageUAT EnvironmentStage = "UAT" + EnvironmentStageQA EnvironmentStage = "QA" + + EnvironmentTypeKubernetes EnvironmentType = "Kubernetes" + EnvironmentTypeCloud EnvironmentType = "Cloud" + EnvironmentTypeOnPrem EnvironmentType = "On-Premises" + EnvironmentTypeOther EnvironmentType = "Other" +) -type PlaybookExecutionDetails struct { - PlaybookID string `json:"playbook_id,omitempty"` - PlaybookName string `json:"playbook_name,omitempty"` - RunID string `json:"run_id,omitempty"` - Status string `json:"status,omitempty"` - Duration string `json:"duration,omitempty"` - Error string `json:"error,omitempty"` -} +type Environment struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + // Optional type of environment, e.g. Kubernetes, Cloud, On-Premises, etc. + EnvironmentType EnvironmentType `json:"type,omitempty"` + // Optional stage or lifecycle phase of the environment, e.g. Development, Staging, Production, UAT, QA, etc. + Stage EnvironmentStage `json:"stage,omitempty"` -func (d PlaybookExecutionDetails) Kind() string { return "PlaybookExecution/v1" } -func (d PlaybookExecutionDetails) MarshalJSON() ([]byte, error) { - type raw PlaybookExecutionDetails - return marshalWithKind(d.Kind(), raw(d)) + Identifier string `json:"identifier,omitempty"` + // Optional tags for additional metadata, e.g. team, cost center, owner, cluster/namespace + Tags map[string]string `json:"tags,omitempty"` } -type ScalingDetails struct { - FromReplicas int `json:"from_replicas,omitempty"` - ToReplicas int `json:"to_replicas,omitempty"` - ResourceType string `json:"resource_type,omitempty"` - Trigger string `json:"trigger,omitempty"` +func (e Environment) Kind() string { return "Environment/v1" } +func (e Environment) MarshalJSON() ([]byte, error) { + type raw Environment + return marshalWithKind(e.Kind(), raw(e)) } -func (d ScalingDetails) Kind() string { return "Scaling/v1" } -func (d ScalingDetails) MarshalJSON() ([]byte, error) { - type raw ScalingDetails - return marshalWithKind(d.Kind(), raw(d)) -} +type TestingType string +type TestingStatus string +type TestingResult string +type Status string -type CertificateDetails struct { - Subject string `json:"subject,omitempty"` - Issuer string `json:"issuer,omitempty"` - NotBefore string `json:"not_before,omitempty"` - NotAfter string `json:"not_after,omitempty"` - Serial string `json:"serial,omitempty"` - DNSNames string `json:"dns_names,omitempty"` +const ( + TestingTypeUnit TestingType = "Unit" + TestingTypeIntegration TestingType = "Integration" + TestingTypeE2E TestingType = "End-to-End" + TestingTypePerformance TestingType = "Performance" + TestingTypeSecurity TestingType = "Security" + + TestingStatusPending TestingStatus = "Pending" + TestingStatusRunning TestingStatus = "Running" + TestingStatusPassed TestingStatus = "Passed" + TestingStatusFailed TestingStatus = "Failed" + TestingStatusSkipped TestingStatus = "Skipped" + TestingStatusError TestingStatus = "Error" + + StatusPending Status = "Pending" + StatusRunning Status = "Running" + StatusTimeout Status = "Timeout" + StatusCompleted Status = "Completed" + StatusFailed Status = "Failed" + StatusApproved Status = "Approved" + StatusRejected Status = "Rejected" + + TestingResultFlaky TestingResult = "Flaky" + TestingResultFailed TestingResult = "Failed" + TestingResultPassed TestingResult = "Passed" +) + +type Event struct { + ID string `json:"id,omitempty"` + URL string `json:"url,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + +func (e Event) Kind() string { return "Event/v1" } +func (e Event) MarshalJSON() ([]byte, error) { + type raw Event + return marshalWithKind(e.Kind(), raw(e)) +} + +type Test struct { + Event `json:",inline"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Type TestingType `json:"type,omitempty"` + Status TestingStatus `json:"status,omitempty"` + Result TestingResult `json:"result,omitempty"` +} + +func (t Test) Kind() string { return "Test/v1" } +func (t Test) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Type TestingType `json:"type,omitempty"` + Status TestingStatus `json:"status,omitempty"` + Result TestingResult `json:"result,omitempty"` + } + return marshalWithKind(t.Kind(), payload{ + rawEvent: rawEvent(t.Event), + Name: t.Name, + Description: t.Description, + Type: t.Type, + Status: t.Status, + Result: t.Result, + }) +} + +type Promotion struct { + Event `json:",inline"` + + // Optional source and target environments for the promotion. If not specified, the promotion is assumed to be within the same environment. + From Environment `json:"from,omitempty"` + To Environment `json:"to,omitempty"` + // Optional source for the promotion, e.g. Git repo, Helm chart, container image, database schema, etc. + Source Source `json:"source,omitempty"` + // Optional version or identifier for the promoted artifact, e.g. image tag, chart version, git commit, database schema version, etc. + Version string `json:"version,omitempty"` + // Optional list of identities who approved the promotion, e.g. users or groups who approved the change, or CI systems that ran tests and checks. + Approvals []Approval `json:"approvals,omitempty"` + // + Artifact string `json:"artifact,omitempty"` +} + +func (p Promotion) Kind() string { return "Promotion/v1" } +func (p Promotion) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + From *Environment `json:"from,omitempty"` + To *Environment `json:"to,omitempty"` + Source *Source `json:"source,omitempty"` + Version string `json:"version,omitempty"` + Approvals []Approval `json:"approvals,omitempty"` + Artifact string `json:"artifact,omitempty"` + } + return marshalWithKind(p.Kind(), payload{ + rawEvent: rawEvent(p.Event), + From: pointerIfNotZero(p.From), + To: pointerIfNotZero(p.To), + Source: pointerIfNotZero(p.Source), + Version: p.Version, + Approvals: p.Approvals, + Artifact: p.Artifact, + }) +} + +type PipelineRun struct { + Event `json:",inline"` + Environment Environment `json:"environment,omitempty"` + Status Status `json:"status,omitempty"` +} + +func (p PipelineRun) Kind() string { return "PipelineRun/v1" } +func (p PipelineRun) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + Environment *Environment `json:"environment,omitempty"` + Status Status `json:"status,omitempty"` + } + return marshalWithKind(p.Kind(), payload{ + rawEvent: rawEvent(p.Event), + Environment: pointerIfNotZero(p.Environment), + Status: p.Status, + }) +} + +type Change struct { + Path string `json:"path,omitempty"` + From map[string]any `json:"from,omitempty"` + To map[string]any `json:"to,omitempty"` + Type string `json:"type,omitempty"` +} + +func (c Change) Kind() string { return "Change/v1" } +func (c Change) MarshalJSON() ([]byte, error) { + type raw Change + return marshalWithKind(c.Kind(), raw(c)) +} + +type ConfigChange struct { + Event `json:",inline"` + Author Identity `json:"author,omitempty"` + Changes []Change `json:"changes,omitempty"` + Environment Environment `json:"environment,omitempty"` + Source Source `json:"source,omitempty"` +} + +func (c ConfigChange) Kind() string { return "ConfigChange/v1" } +func (c ConfigChange) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + Author *Identity `json:"author,omitempty"` + Changes []Change `json:"changes,omitempty"` + Environment *Environment `json:"environment,omitempty"` + Source *Source `json:"source,omitempty"` + } + return marshalWithKind(c.Kind(), payload{ + rawEvent: rawEvent(c.Event), + Author: pointerIfNotZero(c.Author), + Changes: c.Changes, + Environment: pointerIfNotZero(c.Environment), + Source: pointerIfNotZero(c.Source), + }) } -func (d CertificateDetails) Kind() string { return "Certificate/v1" } -func (d CertificateDetails) MarshalJSON() ([]byte, error) { - type raw CertificateDetails - return marshalWithKind(d.Kind(), raw(d)) +type BackupType string + +const ( + BackupTypeDump BackupType = "Dump" + BackupTypeSnapshot BackupType = "Snapshot" + BackupTypeStorageBackup BackupType = "StorageBackup" + BackupTypeOffsite BackupType = "Offsite" + BackupTypeOffAccount BackupType = "OffAccount" + BackupTypeOffRegion BackupType = "OffRegion" +) + +type RestoreType string + +const ( + RestoreTypeClone RestoreType = "Clone" + RestoreTypeSnapshotReset RestoreType = "SnapshotReset" + RestoreTypePointInTime RestoreType = "PointInTime" + RestoreTypeDisasterRecovery RestoreType = "DisasterRecovery" + RestoreTypeTest RestoreType = "RestoreTest" + RestoreTypeData RestoreType = "Data" +) + +type Restore struct { + Event `json:",inline"` + // Optional source and target environments for the restore. If not specified, the restore is assumed to be within the same environment. + From Environment `json:"from,omitempty"` + To Environment `json:"to,omitempty"` + // Optional source for the restore, e.g. Git repo, Helm chart, container image, database schema, etc. + Source Source `json:"source,omitempty"` + // Optional version or identifier for the restored artifact, e.g. image tag, chart version, git commit, database schema version, etc. + Status Status `json:"status,omitempty"` +} + +func (r Restore) Kind() string { return "Restore/v1" } +func (r Restore) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + From *Environment `json:"from,omitempty"` + To *Environment `json:"to,omitempty"` + Source *Source `json:"source,omitempty"` + Status Status `json:"status,omitempty"` + } + return marshalWithKind(r.Kind(), payload{ + rawEvent: rawEvent(r.Event), + From: pointerIfNotZero(r.From), + To: pointerIfNotZero(r.To), + Source: pointerIfNotZero(r.Source), + Status: r.Status, + }) +} + +type Backup struct { + BackupType BackupType `json:"backup_type,omitempty"` + CreatedBy Identity `json:"created_by,omitempty"` + Environment Environment `json:"environment,omitempty"` + Event `json:",inline"` + EndTimestamp string `json:"end,omitempty"` + Status Status `json:"status,omitempty"` + Size string `json:"size,omitempty"` + Delta string `json:"delta,omitempty"` +} + +func (d Backup) Kind() string { return "Backup/v1" } +func (d Backup) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + BackupType BackupType `json:"backup_type,omitempty"` + CreatedBy *Identity `json:"created_by,omitempty"` + Environment *Environment `json:"environment,omitempty"` + EndTimestamp string `json:"end,omitempty"` + Status Status `json:"status,omitempty"` + Size string `json:"size,omitempty"` + Delta string `json:"delta,omitempty"` + } + return marshalWithKind(d.Kind(), payload{ + rawEvent: rawEvent(d.Event), + BackupType: d.BackupType, + CreatedBy: pointerIfNotZero(d.CreatedBy), + Environment: pointerIfNotZero(d.Environment), + EndTimestamp: d.EndTimestamp, + Status: d.Status, + Size: d.Size, + Delta: d.Delta, + }) } -type CostChangeDetails struct { - PreviousCost float64 `json:"previous_cost,omitempty"` - NewCost float64 `json:"new_cost,omitempty"` - Currency string `json:"currency,omitempty"` - Period string `json:"period,omitempty"` - Reason string `json:"reason,omitempty"` +type ScalingDimension string + +const ( + ScalingDimensionCPU ScalingDimension = "CPU" + ScalingDimensionMemory ScalingDimension = "Memory" + ScalingDimensionReplicas ScalingDimension = "Replicas" + ScalingDimensionCustom ScalingDimension = "Custom" +) + +type Dimension struct { + Min string `json:"min,omitempty"` + Max string `json:"max,omitempty"` + Desired string `json:"desired,omitempty"` } -func (d CostChangeDetails) Kind() string { return "CostChange/v1" } -func (d CostChangeDetails) MarshalJSON() ([]byte, error) { - type raw CostChangeDetails +func (d Dimension) Kind() string { return "Dimension/v1" } +func (d Dimension) MarshalJSON() ([]byte, error) { + type raw Dimension return marshalWithKind(d.Kind(), raw(d)) } -type PipelineRunDetails struct { - PipelineID string `json:"pipeline_id,omitempty"` - PipelineName string `json:"pipeline_name,omitempty"` - RunID string `json:"run_id,omitempty"` - RunNumber int `json:"run_number,omitempty"` - Branch string `json:"branch,omitempty"` - Status string `json:"status,omitempty"` - Duration string `json:"duration,omitempty"` - Error string `json:"error,omitempty"` +type Scale struct { + Dimension ScalingDimension `json:"dimension,omitempty"` + PreviousValue Dimension `json:"previous_value,omitempty"` + Value Dimension `json:"value,omitempty"` } -func (d PipelineRunDetails) Kind() string { return "PipelineRun/v1" } -func (d PipelineRunDetails) MarshalJSON() ([]byte, error) { - type raw PipelineRunDetails - return marshalWithKind(d.Kind(), raw(d)) +func (d Scale) Kind() string { return "Scale/v1" } +func (d Scale) MarshalJSON() ([]byte, error) { + type payload struct { + Dimension ScalingDimension `json:"dimension,omitempty"` + PreviousValue *Dimension `json:"previous_value,omitempty"` + Value *Dimension `json:"value,omitempty"` + } + return marshalWithKind(d.Kind(), payload{ + Dimension: d.Dimension, + PreviousValue: pointerIfNotZero(d.PreviousValue), + Value: pointerIfNotZero(d.Value), + }) } diff --git a/types/config_changes_test.go b/types/config_changes_test.go index f072d7144..952c266be 100644 --- a/types/config_changes_test.go +++ b/types/config_changes_test.go @@ -2,95 +2,368 @@ package types import ( "encoding/json" + "slices" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +type kinded interface { + Kind() string +} + +func expectKind(v kinded) map[string]any { + data, err := json.Marshal(v) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal(v.Kind())) + return m +} + var _ = Describe("ConfigChangeDetails", func() { - It("should inject kind into DeploymentDetails", func() { - d := DeploymentDetails{PreviousImage: "v1.2.3", NewImage: "v1.2.4", Container: "app"} - data, err := json.Marshal(d) - Expect(err).ToNot(HaveOccurred()) + DescribeTable("current detail payloads implement ConfigChangeDetail", + func(d ConfigChangeDetail, expectedFields map[string]any) { + m := expectKind(d) + for key, val := range expectedFields { + Expect(m[key]).To(Equal(val)) + } + }, + Entry("UserChange", UserChangeDetails{UserName: "alice"}, map[string]any{ + "user_name": "alice", + }), + Entry("Screenshot", ScreenshotDetails{URL: "https://example.com"}, map[string]any{ + "url": "https://example.com", + }), + Entry("PermissionChange", PermissionChangeDetails{RoleName: "admin"}, map[string]any{ + "role_name": "admin", + }), + Entry("GroupMembership", GroupMembership{ + Group: Identity{Name: "platform-admins", Type: IdentityTypeGroup}, + Member: Identity{Name: "alice", Type: IdentityTypeUser}, + Action: GroupMembershipActionAdded, + }, map[string]any{ + "action": string(GroupMembershipActionAdded), + }), + Entry("Backup", Backup{Status: StatusCompleted, Size: "4.2GB"}, map[string]any{ + "status": string(StatusCompleted), + "size": "4.2GB", + }), + Entry("Scale", Scale{ + Dimension: ScalingDimensionReplicas, + PreviousValue: Dimension{Desired: "2"}, + Value: Dimension{Desired: "5"}, + }, map[string]any{ + "dimension": string(ScalingDimensionReplicas), + }), + ) - var m map[string]any - Expect(json.Unmarshal(data, &m)).To(Succeed()) - Expect(m["kind"]).To(Equal("Deployment/v1")) - Expect(m["previous_image"]).To(Equal("v1.2.3")) - Expect(m["new_image"]).To(Equal("v1.2.4")) - Expect(m["container"]).To(Equal("app")) - }) + DescribeTable("exported structs inject kind", + func(v kinded, expectedFields map[string]any) { + m := expectKind(v) + for key, val := range expectedFields { + Expect(m[key]).To(Equal(val)) + } + }, + Entry("Identity", Identity{Name: "alice"}, map[string]any{ + "name": "alice", + }), + Entry("Approval", Approval{ + Event: Event{ID: "evt-1"}, + SubmittedBy: &Identity{Name: "submitter"}, + Approver: &Identity{Name: "approver"}, + Stage: ApprovalStageManual, + Status: ApprovalStatusApproved, + }, map[string]any{ + "id": "evt-1", + "stage": string(ApprovalStageManual), + "status": string(ApprovalStatusApproved), + }), + Entry("GitSource", GitSource{URL: "https://example.com/repo.git"}, map[string]any{ + "url": "https://example.com/repo.git", + }), + Entry("HelmSource", HelmSource{ChartName: "api"}, map[string]any{ + "chart_name": "api", + }), + Entry("ImageSource", ImageSource{ImageName: "backend"}, map[string]any{ + "image": "backend", + }), + Entry("DatabaseSource", DatabaseSource{Name: "appdb"}, map[string]any{ + "name": "appdb", + }), + Entry("Source", Source{ + Git: &GitSource{URL: "https://example.com/repo.git"}, + Path: "deploy/app", + }, map[string]any{ + "path": "deploy/app", + }), + Entry("Environment", Environment{Name: "prod"}, map[string]any{ + "name": "prod", + }), + Entry("Event", Event{ID: "evt-2"}, map[string]any{ + "id": "evt-2", + }), + Entry("Test", Test{ + Event: Event{ID: "evt-3"}, + Name: "smoke", + Type: TestingTypeE2E, + Status: TestingStatusPassed, + Result: TestingResultPassed, + }, map[string]any{ + "id": "evt-3", + "name": "smoke", + "type": string(TestingTypeE2E), + "status": string(TestingStatusPassed), + "result": string(TestingResultPassed), + }), + Entry("Promotion", Promotion{ + Event: Event{ID: "evt-4"}, + From: Environment{Name: "staging"}, + To: Environment{Name: "prod"}, + Source: Source{Git: &GitSource{URL: "https://example.com/repo.git"}}, + Version: "v1.2.3", + }, map[string]any{ + "id": "evt-4", + "version": "v1.2.3", + }), + Entry("PipelineRun", PipelineRun{ + Event: Event{ID: "evt-5"}, + Environment: Environment{Name: "prod"}, + Status: StatusRunning, + }, map[string]any{ + "id": "evt-5", + "status": string(StatusRunning), + }), + Entry("Change", Change{ + Path: ".spec.replicas", + From: map[string]any{"desired": "2"}, + To: map[string]any{"desired": "3"}, + Type: "update", + }, map[string]any{ + "path": ".spec.replicas", + "type": "update", + }), + Entry("ConfigChange", ConfigChange{ + Event: Event{ID: "evt-6"}, + Author: Identity{Name: "alice"}, + Changes: []Change{{Path: ".spec.replicas", Type: "update"}}, + Environment: Environment{Name: "prod"}, + Source: Source{Git: &GitSource{URL: "https://example.com/repo.git"}}, + }, map[string]any{ + "id": "evt-6", + }), + Entry("Restore", Restore{ + Event: Event{ID: "evt-7"}, + From: Environment{Name: "backup"}, + To: Environment{Name: "prod"}, + Source: Source{Database: &DatabaseSource{Name: "appdb"}}, + Status: StatusCompleted, + }, map[string]any{ + "id": "evt-7", + "status": string(StatusCompleted), + }), + Entry("Dimension", Dimension{Desired: "3"}, map[string]any{ + "desired": "3", + }), + ) - It("should inject kind into BackupDetails", func() { - d := BackupDetails{Status: "success", Size: "4.2GB"} - data, err := json.Marshal(d) + It("preserves embedded event fields and nested kinds", func() { + data, err := json.Marshal(ConfigChange{ + Event: Event{ID: "evt-8", Timestamp: "2026-04-10T12:00:00Z"}, + Author: Identity{Name: "alice"}, + Changes: []Change{{ + Path: ".spec.replicas", + Type: "update", + }}, + Environment: Environment{Name: "prod", Stage: EnvironmentStageProduction}, + Source: Source{ + Git: &GitSource{ + URL: "https://example.com/repo.git", + }, + }, + }) Expect(err).ToNot(HaveOccurred()) var m map[string]any Expect(json.Unmarshal(data, &m)).To(Succeed()) - Expect(m["kind"]).To(Equal("Backup/v1")) - Expect(m["status"]).To(Equal("success")) - Expect(m["size"]).To(Equal("4.2GB")) + Expect(m["kind"]).To(Equal("ConfigChange/v1")) + Expect(m["id"]).To(Equal("evt-8")) + Expect(m["timestamp"]).To(Equal("2026-04-10T12:00:00Z")) + + author := m["author"].(map[string]any) + Expect(author["kind"]).To(Equal("Identity/v1")) + Expect(author["name"]).To(Equal("alice")) + + environment := m["environment"].(map[string]any) + Expect(environment["kind"]).To(Equal("Environment/v1")) + Expect(environment["name"]).To(Equal("prod")) + + source := m["source"].(map[string]any) + Expect(source["kind"]).To(Equal("Source/v1")) + git := source["git"].(map[string]any) + Expect(git["kind"]).To(Equal("GitSource/v1")) + + changes := m["changes"].([]any) + Expect(changes).To(HaveLen(1)) + change := changes[0].(map[string]any) + Expect(change["kind"]).To(Equal("Change/v1")) + Expect(change["path"]).To(Equal(".spec.replicas")) }) - It("should inject kind into empty struct", func() { - d := RollbackDetails{} - data, err := json.Marshal(d) + It("emits nested identity kinds for GroupMembership", func() { + data, err := json.Marshal(GroupMembership{ + Group: Identity{ID: "g-1", Name: "platform-admins", Type: IdentityTypeGroup}, + Member: Identity{ID: "u-1", Name: "alice", Type: IdentityTypeUser}, + Action: GroupMembershipActionAdded, + Tenant: "acme", + }) Expect(err).ToNot(HaveOccurred()) var m map[string]any Expect(json.Unmarshal(data, &m)).To(Succeed()) - Expect(m["kind"]).To(Equal("Rollback/v1")) - Expect(m).To(HaveLen(1)) + Expect(m["kind"]).To(Equal("GroupMembership/v1")) + Expect(m["action"]).To(Equal(string(GroupMembershipActionAdded))) + Expect(m["tenant"]).To(Equal("acme")) + + group := m["group"].(map[string]any) + Expect(group["kind"]).To(Equal("Identity/v1")) + Expect(group["name"]).To(Equal("platform-admins")) + Expect(group["type"]).To(Equal(string(IdentityTypeGroup))) + + member := m["member"].(map[string]any) + Expect(member["kind"]).To(Equal("Identity/v1")) + Expect(member["name"]).To(Equal("alice")) + Expect(member["type"]).To(Equal(string(IdentityTypeUser))) }) - It("should inject kind into ScalingDetails with numeric fields", func() { - d := ScalingDetails{FromReplicas: 2, ToReplicas: 5, ResourceType: "Deployment"} - data, err := json.Marshal(d) + It("omits zero-value nested struct fields from outer payloads", func() { + data, err := json.Marshal(PipelineRun{ + Event: Event{ID: "evt-9"}, + Status: StatusPending, + }) Expect(err).ToNot(HaveOccurred()) var m map[string]any Expect(json.Unmarshal(data, &m)).To(Succeed()) - Expect(m["kind"]).To(Equal("Scaling/v1")) - Expect(m["from_replicas"]).To(BeNumerically("==", 2)) - Expect(m["to_replicas"]).To(BeNumerically("==", 5)) + Expect(m["kind"]).To(Equal("PipelineRun/v1")) + Expect(m["id"]).To(Equal("evt-9")) + Expect(m).ToNot(HaveKey("environment")) }) - It("should inject kind into CostChangeDetails with float fields", func() { - d := CostChangeDetails{PreviousCost: 100.50, NewCost: 125.75, Currency: "USD"} - data, err := json.Marshal(d) - Expect(err).ToNot(HaveOccurred()) + It("registers all exported structs for kind lookup", func() { + kinds := make([]string, 0, len(configChangeDetailTypes)) + for _, candidate := range configChangeDetailTypes { + kinds = append(kinds, candidate.Kind()) + } + slices.Sort(kinds) - var m map[string]any - Expect(json.Unmarshal(data, &m)).To(Succeed()) - Expect(m["kind"]).To(Equal("CostChange/v1")) - Expect(m["previous_cost"]).To(BeNumerically("~", 100.50, 0.01)) - Expect(m["new_cost"]).To(BeNumerically("~", 125.75, 0.01)) + Expect(kinds).To(Equal([]string{ + "Approval/v1", + "Backup/v1", + "Change/v1", + "ConfigChange/v1", + "DatabaseSource/v1", + "Dimension/v1", + "Environment/v1", + "Event/v1", + "GitSource/v1", + "GroupMembership/v1", + "HelmSource/v1", + "Identity/v1", + "ImageSource/v1", + "PermissionChange/v1", + "PipelineRun/v1", + "Promotion/v1", + "Restore/v1", + "Scale/v1", + "Screenshot/v1", + "Source/v1", + "Test/v1", + "UserChange/v1", + })) }) - DescribeTable("all detail types implement ConfigChangeDetail", - func(d ConfigChangeDetail, expectedKind string) { - Expect(d.Kind()).To(Equal(expectedKind)) + DescribeTable("UnmarshalChangeDetails returns the matching registered type", + func(in ConfigChangeDetail, expected any) { + raw, err := json.Marshal(in) + Expect(err).ToNot(HaveOccurred()) - data, err := json.Marshal(d) + got, err := UnmarshalChangeDetails(raw) Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeAssignableToTypeOf(expected)) - var m map[string]any - Expect(json.Unmarshal(data, &m)).To(Succeed()) - Expect(m["kind"]).To(Equal(expectedKind)) + reraw, err := json.Marshal(got) + Expect(err).ToNot(HaveOccurred()) + Expect(reraw).To(MatchJSON(raw)) }, - Entry("UserChange", UserChangeDetails{UserName: "alice"}, "UserChange/v1"), - Entry("Screenshot", ScreenshotDetails{URL: "https://example.com"}, "Screenshot/v1"), - Entry("PermissionChange", PermissionChangeDetails{RoleName: "admin"}, "PermissionChange/v1"), - Entry("Deployment", DeploymentDetails{NewImage: "v2"}, "Deployment/v1"), - Entry("Promotion", PromotionDetails{ToEnvironment: "prod"}, "Promotion/v1"), - Entry("Approval", ApprovalDetails{ApprovedBy: "alice"}, "Approval/v1"), - Entry("Rollback", RollbackDetails{ToVersion: "v1"}, "Rollback/v1"), - Entry("Backup", BackupDetails{Status: "success"}, "Backup/v1"), - Entry("PlaybookExecution", PlaybookExecutionDetails{PlaybookName: "restart"}, "PlaybookExecution/v1"), - Entry("Scaling", ScalingDetails{ToReplicas: 3}, "Scaling/v1"), - Entry("Certificate", CertificateDetails{Subject: "*.example.com"}, "Certificate/v1"), - Entry("CostChange", CostChangeDetails{NewCost: 50}, "CostChange/v1"), - Entry("PipelineRun", PipelineRunDetails{Branch: "main"}, "PipelineRun/v1"), + Entry("UserChange", UserChangeDetails{UserName: "alice"}, UserChangeDetails{}), + Entry("Screenshot", ScreenshotDetails{URL: "https://example.com"}, ScreenshotDetails{}), + Entry("PermissionChange", PermissionChangeDetails{RoleName: "admin"}, PermissionChangeDetails{}), + Entry("GroupMembership", GroupMembership{ + Group: Identity{ID: "g-1", Type: IdentityTypeGroup}, + Member: Identity{ID: "u-1", Type: IdentityTypeUser}, + Action: GroupMembershipActionRemoved, + }, GroupMembership{}), + Entry("Identity", Identity{Name: "alice"}, Identity{}), + Entry("Approval", Approval{ + Event: Event{ID: "evt-10"}, + SubmittedBy: &Identity{Name: "alice"}, + Status: ApprovalStatusApproved, + }, Approval{}), + Entry("GitSource", GitSource{URL: "https://example.com/repo.git"}, GitSource{}), + Entry("HelmSource", HelmSource{ChartName: "api"}, HelmSource{}), + Entry("ImageSource", ImageSource{ImageName: "backend"}, ImageSource{}), + Entry("DatabaseSource", DatabaseSource{Name: "appdb"}, DatabaseSource{}), + Entry("Source", Source{ + Git: &GitSource{URL: "https://example.com/repo.git"}, + Path: "deploy/app", + }, Source{}), + Entry("Environment", Environment{Name: "prod"}, Environment{}), + Entry("Event", Event{ID: "evt-11"}, Event{}), + Entry("Test", Test{ + Event: Event{ID: "evt-12"}, + Name: "smoke", + }, Test{}), + Entry("Promotion", Promotion{ + Event: Event{ID: "evt-13"}, + Version: "v1.2.3", + }, Promotion{}), + Entry("PipelineRun", PipelineRun{ + Event: Event{ID: "evt-14"}, + Status: StatusRunning, + }, PipelineRun{}), + Entry("Change", Change{Path: ".spec.replicas", Type: "update"}, Change{}), + Entry("ConfigChange", ConfigChange{ + Event: Event{ID: "evt-15"}, + Author: Identity{Name: "alice"}, + }, ConfigChange{}), + Entry("Restore", Restore{ + Event: Event{ID: "evt-16"}, + Status: StatusCompleted, + }, Restore{}), + Entry("Backup", Backup{Status: StatusCompleted, Size: "4.2GB"}, Backup{}), + Entry("Dimension", Dimension{Desired: "3"}, Dimension{}), + Entry("Scale", Scale{Dimension: ScalingDimensionReplicas, Value: Dimension{Desired: "3"}}, Scale{}), ) + + It("returns nil for empty and null details payloads", func() { + got, err := UnmarshalChangeDetails(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeNil()) + + got, err = UnmarshalChangeDetails([]byte("null")) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeNil()) + }) + + It("returns an error for an unknown kind", func() { + _, err := UnmarshalChangeDetails([]byte(`{"kind":"Nope/v1"}`)) + Expect(err).To(MatchError(ContainSubstring("unknown config change detail kind"))) + }) + + It("returns an error for invalid JSON", func() { + _, err := UnmarshalChangeDetails([]byte(`{`)) + Expect(err).To(MatchError(ContainSubstring("decode config change details envelope"))) + }) }) From f6bad41577e58536b47a9404ce727d3d999d0c59 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 12:02:21 +0300 Subject: [PATCH 07/12] feat(auth): add bearer token fallback for azure connections Support static bearer token authentication for Azure connections when client credentials are not provided. Implements TokenCredential interface with a static token provider that defaults expiration to 1 hour. --- connection/azure.go | 27 ++++++++++++++++++++ connection/merge_test.go | 54 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/connection/azure.go b/connection/azure.go index 89b5e4a6c..f9d6f2488 100644 --- a/connection/azure.go +++ b/connection/azure.go @@ -1,7 +1,11 @@ package connection import ( + "context" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" @@ -13,6 +17,11 @@ type AzureConnection struct { ClientID *types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"` ClientSecret *types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"` TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty"` + + // bearerToken is populated from the referenced connection's + // Properties["bearer"] when the connection has no Username/Password set. + // It is an unexported fallback — not part of the inline YAML surface. + bearerToken string `json:"-" yaml:"-"` } // HydrateConnection attempts to find the connection by name @@ -33,6 +42,10 @@ func (g *AzureConnection) HydrateConnection(ctx ConnectionContext) error { if g.TenantID == "" { g.TenantID = connection.Properties["tenant"] } + + if connection.Username == "" && connection.Password == "" { + g.bearerToken = connection.Properties["bearer"] + } } return nil @@ -45,6 +58,9 @@ func (g *AzureConnection) FromModel(connection models.Connection) { if tenantID, ok := connection.Properties["tenant"]; ok { g.TenantID = tenantID } + if connection.Username == "" && connection.Password == "" { + g.bearerToken = connection.Properties["bearer"] + } } func (g AzureConnection) ToModel() models.Connection { @@ -60,5 +76,16 @@ func (g AzureConnection) ToModel() models.Connection { } func (g *AzureConnection) TokenCredential() (azcore.TokenCredential, error) { + if (g.ClientID == nil || g.ClientID.IsEmpty()) && + (g.ClientSecret == nil || g.ClientSecret.IsEmpty()) && + g.bearerToken != "" { + return staticTokenCredential{token: g.bearerToken}, nil + } return azidentity.NewClientSecretCredential(g.TenantID, g.ClientID.String(), g.ClientSecret.String(), nil) } + +type staticTokenCredential struct{ token string } + +func (s staticTokenCredential) GetToken(_ context.Context, _ policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: s.token, ExpiresOn: time.Now().Add(time.Hour)}, nil +} diff --git a/connection/merge_test.go b/connection/merge_test.go index a57a47815..501d14772 100644 --- a/connection/merge_test.go +++ b/connection/merge_test.go @@ -5,6 +5,8 @@ import ( "strconv" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" "github.com/onsi/gomega" @@ -524,6 +526,58 @@ func TestAzureConnectionFallsBackToConnection(t *testing.T) { g.Expect(conn.TenantID).To(gomega.Equal("conn-tenant")) } +func TestAzureConnectionUsesBearerWhenClientSecretAbsent(t *testing.T) { + g := gomega.NewWithT(t) + + azConn := &models.Connection{ + Type: models.ConnectionTypeAzure, + Properties: types.JSONStringMap{ + "bearer": "test-bearer-token", + }, + } + ctx := mockConnectionContext{Context: gocontext.Background(), connection: azConn} + + conn := AzureConnection{ConnectionName: "connection://azure"} + err := conn.HydrateConnection(ctx) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(conn.ClientID.ValueStatic).To(gomega.BeEmpty()) + g.Expect(conn.ClientSecret.ValueStatic).To(gomega.BeEmpty()) + + cred, err := conn.TokenCredential() + g.Expect(err).ToNot(gomega.HaveOccurred()) + token, err := cred.GetToken(gocontext.Background(), policy.TokenRequestOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(token.Token).To(gomega.Equal("test-bearer-token")) +} + +func TestAzureConnectionPrefersClientSecretOverBearer(t *testing.T) { + g := gomega.NewWithT(t) + + azConn := &models.Connection{ + Type: models.ConnectionTypeAzure, + Username: "conn-client-id", + Password: "conn-client-secret", + Properties: types.JSONStringMap{ + "tenant": "conn-tenant", + "bearer": "should-be-ignored", + }, + } + ctx := mockConnectionContext{Context: gocontext.Background(), connection: azConn} + + conn := AzureConnection{ConnectionName: "connection://azure"} + err := conn.HydrateConnection(ctx) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(conn.ClientID.ValueStatic).To(gomega.Equal("conn-client-id")) + g.Expect(conn.ClientSecret.ValueStatic).To(gomega.Equal("conn-client-secret")) + + cred, err := conn.TokenCredential() + g.Expect(err).ToNot(gomega.HaveOccurred()) + _, isStatic := cred.(staticTokenCredential) + g.Expect(isStatic).To(gomega.BeFalse(), "should return ClientSecretCredential, not bearer") + _, isClientSecret := cred.(*azidentity.ClientSecretCredential) + g.Expect(isClientSecret).To(gomega.BeTrue()) +} + func TestOpensearchConnectionInlineOverridesConnection(t *testing.T) { g := gomega.NewWithT(t) From f87d69da446c66f61bea716beb2857c8abb7c3fb Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 12:03:11 +0300 Subject: [PATCH 08/12] feat(api): add WithPrefix method to retrieve properties by prefix Enables filtering hierarchical properties by key prefix while maintaining precedence order (CLI/env > local > parent chain > global DB). Returned keys have the prefix stripped for easier consumption. --- context/properties.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/context/properties.go b/context/properties.go index d0b09bdbb..abf525be9 100644 --- a/context/properties.go +++ b/context/properties.go @@ -195,6 +195,46 @@ func (h HierarchicalProperties) getProperty(key string) (string, bool) { return v, ok } +// WithPrefix returns all properties whose key begins with prefix, resolved +// with the normal precedence (CLI/env → local → parent chain → global DB). +// The returned keys have the prefix STRIPPED. Later sources in the chain +// override earlier ones, matching getProperty's semantics. +func (h HierarchicalProperties) WithPrefix(prefix string) map[string]string { + out := make(map[string]string) + + // Walk global first (lowest precedence). + for k, v := range h.global { + if after, ok := strings.CutPrefix(k, prefix); ok { + out[after] = v + } + } + + // Then walk the parent chain from root to this node (so this node + // overrides its ancestors). + var chain []HierarchicalProperties + cur := &h + for cur != nil { + chain = append(chain, *cur) + cur = cur.parent + } + for i := len(chain) - 1; i >= 0; i-- { + for k, v := range chain[i].local { + if after, ok := strings.CutPrefix(k, prefix); ok { + out[after] = v + } + } + } + + // Finally, CLI/env takes highest precedence. + for k, v := range properties.Global.GetAll() { + if after, ok := strings.CutPrefix(k, prefix); ok { + out[after] = v + } + } + + return out +} + func (k Context) globalProperties() Properties { if val, ok := propertyCache.Get("global"); ok { return val.(map[string]string) From b24b9b5ad96a92909e60c1895bd05b56f2aad3ea Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 12:03:42 +0300 Subject: [PATCH 09/12] feat(db): route postgres notices and warnings to application logger Add NewGormFromPool to share pgxpool between GORM and direct pgx users, enabling server-side RAISE NOTICE/WARNING messages to flow through ConnConfig.OnNotice. Add ApplySessionProperties to route context properties to Postgres SET LOCAL commands. --- db.go | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/db.go b/db.go index 4c9c59db3..c75c6aff3 100644 --- a/db.go +++ b/db.go @@ -5,10 +5,13 @@ import ( "database/sql" "flag" "fmt" + "sort" + "strings" "time" "github.com/exaring/otelpgx" "github.com/flanksource/commons/logger" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/stdlib" gormpostgres "gorm.io/driver/postgres" @@ -78,6 +81,84 @@ func NewGorm(connection string, config *gorm.Config) (*gorm.DB, error) { return Gorm, nil } +// NewGormFromPool creates a Gorm DB that reuses an existing *pgxpool.Pool. +// This is the preferred path for the main application: it shares the same +// connection pool (and therefore the ConnConfig.OnNotice handler, pgx +// Tracer, and MaxConns) with any direct pgxpool users on the same context. +// +// In particular, without sharing the pool, RAISE NOTICE / RAISE WARNING +// messages emitted by server-side functions are invisible to Go callers +// using ctx.DB() (GORM) because the stdlib driver creates its own pgx +// config with no notice handler. +// +// stdlib.OpenDBFromPool automatically sets db.SetMaxIdleConns(0), so GORM +// will not hoard connections from the pool. +func NewGormFromPool(pool *pgxpool.Pool, config *gorm.Config) (*gorm.DB, error) { + db := stdlib.OpenDBFromPool(pool) + + gormDB, err := gorm.Open( + gormpostgres.New(gormpostgres.Config{Conn: db}), + config, + ) + if err != nil { + return nil, err + } + + if err := gormDB.Use(tracing.NewPlugin()); err != nil { + return nil, fmt.Errorf("error setting up tracing: %w", err) + } + + return gormDB, nil +} + +// SessionPropertyPrefix is the property-name prefix used to route values +// into Postgres session/transaction-local settings. A property +// "postgres.session.eu_debug.enabled=on" translates to +// `SET LOCAL eu_debug.enabled = 'on'` inside the provided transaction. +const SessionPropertyPrefix = "postgres.session." + +// ApplySessionProperties runs `SET LOCAL = ''` inside the +// supplied GORM transaction for every property in `ctx.Properties()` whose +// key starts with SessionPropertyPrefix. The prefix is stripped before the +// SET is issued. Values are passed as text; Postgres will coerce them as +// needed by the GUC's type. +// +// The transaction MUST be an open tx (returned by `db.Begin()` or +// `db.WithContext(...).Begin()`); `SET LOCAL` outside a tx has no effect. +// +// Typical usage: +// +// tx := ctx.DB().Begin() +// if err := duty.ApplySessionProperties(ctx, tx); err != nil { ... } +// defer tx.Rollback() // or commit +// ... +func ApplySessionProperties(ctx dutyContext.Context, tx *gorm.DB) error { + if tx == nil { + return fmt.Errorf("ApplySessionProperties: nil transaction") + } + settings := ctx.Properties().WithPrefix(SessionPropertyPrefix) + if len(settings) == 0 { + return nil + } + // Sort keys for deterministic apply order (helps tests and log output). + keys := make([]string, 0, len(settings)) + for k := range settings { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := settings[k] + // set_config(setting, new_value, is_local) is the parameterizable + // equivalent of `SET LOCAL = ` — it accepts the GUC + // name as a text parameter, avoiding any need to escape/quote the + // identifier ourselves. + if err := tx.Exec("SELECT set_config(?, ?, true)", k, v).Error; err != nil { + return fmt.Errorf("failed to set session property %q=%q: %w", k, v, err) + } + } + return nil +} + func getConnection(connection string) (string, error) { pgxConfig, err := drivers.ParseURL(connection) if err != nil { @@ -121,6 +202,28 @@ func NewPgxPool(connection string) (*pgxpool.Pool, error) { }), ) + // Route Postgres NOTICE / WARNING messages (emitted via `RAISE NOTICE` + // or `RAISE WARNING` from server-side functions) to the application + // logger so server-side debug output is visible without needing to + // attach psql. Severity → log level mapping mirrors Postgres semantics. + config.ConnConfig.OnNotice = func(_ *pgconn.PgConn, n *pgconn.Notice) { + if n == nil { + return + } + switch strings.ToUpper(n.Severity) { + case "ERROR", "FATAL", "PANIC": + logger.Errorf("pg %s: %s", n.Severity, n.Message) + case "WARNING": + logger.Warnf("pg %s: %s", n.Severity, n.Message) + case "NOTICE", "INFO": + logger.Infof("pg %s: %s", n.Severity, n.Message) + case "LOG", "DEBUG": + logger.Debugf("pg %s: %s", n.Severity, n.Message) + default: + logger.Infof("pg %s: %s", n.Severity, n.Message) + } + } + // prevent deadlocks from concurrent queries if config.MaxConns < 20 { config.MaxConns = 20 @@ -217,7 +320,10 @@ func SetupDB(config api.Config) (gormDB *gorm.DB, pgxpool *pgxpool.Pool, err err cfg.Logger = dutyGorm.NewSqlLogger(logger.GetLogger(config.LogName)) } - gormDB, err = NewGorm(config.ConnectionString, cfg) + // Share the pgxpool with GORM so Notice/Warning messages emitted by + // server-side functions (RAISE NOTICE / RAISE WARNING) flow into the + // pool's ConnConfig.OnNotice handler installed by NewPgxPool. + gormDB, err = NewGormFromPool(pgxpool, cfg) if err != nil { return } From 485b8e34a443412f24da9ab41e2bfe942f8bee2b Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 12:49:36 +0300 Subject: [PATCH 10/12] fix(db): fix external user/group/role merge logic and add debug instrumentation Add alias normalization before edge detection to match trigger behavior. Pre-soft-delete live losers before inserting winners to prevent unique index violations. Fix external_role_id selection in config_access view. Add comprehensive debug logging infrastructure for troubleshooting merge operations. --- db.go | 4 +- views/038_config_access.sql | 2 +- views/045_merge_external_entities.sql | 201 ++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 3 deletions(-) diff --git a/db.go b/db.go index c75c6aff3..fc1163c62 100644 --- a/db.go +++ b/db.go @@ -113,8 +113,8 @@ func NewGormFromPool(pool *pgxpool.Pool, config *gorm.Config) (*gorm.DB, error) // SessionPropertyPrefix is the property-name prefix used to route values // into Postgres session/transaction-local settings. A property -// "postgres.session.eu_debug.enabled=on" translates to -// `SET LOCAL eu_debug.enabled = 'on'` inside the provided transaction. +// "postgres.session.debug_log.enabled=on" translates to +// `SET LOCAL debug_log.enabled = 'on'` inside the provided transaction. const SessionPropertyPrefix = "postgres.session." // ApplySessionProperties runs `SET LOCAL = ''` inside the diff --git a/views/038_config_access.sql b/views/038_config_access.sql index 02aa2826e..74e43bd2e 100644 --- a/views/038_config_access.sql +++ b/views/038_config_access.sql @@ -11,7 +11,7 @@ SELECT config_access.config_id, external_user_groups.external_user_id, config_access.external_group_id AS external_group_id, - NULL AS external_role_id, + config_access.external_role_id, config_access.created_at, config_access.deleted_at, config_access.deleted_by, diff --git a/views/045_merge_external_entities.sql b/views/045_merge_external_entities.sql index 96a908317..0cc42af19 100644 --- a/views/045_merge_external_entities.sql +++ b/views/045_merge_external_entities.sql @@ -1,12 +1,74 @@ +-- _debug_log: generic debug helper for instrumenting plpgsql functions. +-- Gated on session setting `debug_log.enabled = 'on'`. Emits the step + +-- detail as a `RAISE NOTICE` so the output survives savepoint rollback +-- (NOTICE messages stream to the client out-of-band and are not rolled +-- back by transaction/savepoint aborts — unlike INSERTs into any table, +-- temp or otherwise). +-- +-- The NOTICE payload is prefixed with a sentinel `_DEBUG_LOG:` so callers +-- (both psql and the Go dump helper) can grep for it among other notices. +-- +-- Usage: +-- SET LOCAL debug_log.enabled = 'on'; +-- ... call the instrumented function inside a savepoint ... +-- -- NOTICE lines appear in the client stream even if the call failed. +CREATE OR REPLACE FUNCTION _debug_log(p_step TEXT, p_detail JSONB) RETURNS void AS $$ +BEGIN + IF current_setting('debug_log.enabled', true) IS DISTINCT FROM 'on' THEN + RETURN; + END IF; + RAISE NOTICE '_DEBUG_LOG: % %', p_step, COALESCE(p_detail::text, 'null'); +END; +$$ LANGUAGE plpgsql; + + -- merge_and_upsert_external_users: detects alias overlaps within temp table -- and against live table, remaps FKs, merges aliases, soft-deletes losers, upserts survivors. -- Returns (loser_id, winner_id) pairs for caller cache eviction. CREATE OR REPLACE FUNCTION merge_and_upsert_external_users(p_temp_table TEXT) RETURNS TABLE(loser_id UUID, winner_id UUID) AS $$ +DECLARE + v_debug BOOLEAN := current_setting('debug_log.enabled', true) = 'on'; + v_row_count BIGINT; + v_edges_total BIGINT; + v_temp_live_count BIGINT; + v_temp_live_no_del_filter BIGINT; + v_temp_live_null_aliases BIGINT; + v_temp_live_id_eq BIGINT; + v_sample JSONB; BEGIN LOCK TABLE config_access, access_reviews, config_access_logs, external_user_groups, external_users, external_groups, external_roles IN SHARE ROW EXCLUSIVE MODE; + IF v_debug THEN + EXECUTE format('SELECT count(*) FROM %I', p_temp_table) INTO v_row_count; + EXECUTE format('SELECT jsonb_agg(to_jsonb(t)) FROM (SELECT * FROM %I LIMIT 5) t', p_temp_table) INTO v_sample; + PERFORM _debug_log('entry', jsonb_build_object( + 'temp_table', p_temp_table, + 'row_count', v_row_count, + 'sample', v_sample + )); + END IF; + + -- Step 0: Normalize the temp table's aliases to match the normalization + -- that the `normalize_aliases` BEFORE INSERT/UPDATE trigger applies to + -- rows stored in external_users (lowercase + sorted + distinct). The + -- edge-build query below uses `tmp.aliases && live.aliases`, which is + -- byte-exact, so any mixed-case alias in the temp table would miss + -- matches against live rows whose aliases the trigger has already + -- normalized. Normalizing up-front keeps the && join symmetric and + -- reliable. + EXECUTE format(' + UPDATE %1$I SET + aliases = ARRAY(SELECT DISTINCT LOWER(elem) FROM unnest(aliases) AS elem ORDER BY LOWER(elem)) + WHERE aliases IS NOT NULL + ', p_temp_table); + + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step0_normalize_temp_aliases', jsonb_build_object('rows_affected', v_row_count)); + END IF; + -- Step 1: Build undirected edge list from alias overlaps EXECUTE format(' CREATE TEMP TABLE _eu_edges ON COMMIT DROP AS @@ -19,7 +81,69 @@ BEGIN ON tmp.aliases && live.aliases AND tmp.id != live.id AND live.deleted_at IS NULL ', p_temp_table); + IF v_debug THEN + -- Post Step 1: edge counts + sample edges. + SELECT count(*) INTO v_edges_total FROM _eu_edges; + PERFORM _debug_log('step1_edges', jsonb_build_object( + 'edge_count', v_edges_total, + 'sample_edges', (SELECT jsonb_agg(jsonb_build_object('id1', id1, 'id2', id2)) + FROM (SELECT * FROM _eu_edges LIMIT 20) e) + )); + + -- Diagnostic re-query: same join with variations to pinpoint which filter + -- is dropping overlaps. Uses unnest + = ANY to bypass the && operator in + -- case &&'s behavior is the culprit. + EXECUTE format(' + SELECT count(*) FROM %1$I tmp JOIN external_users live + ON tmp.aliases && live.aliases AND tmp.id != live.id AND live.deleted_at IS NULL + ', p_temp_table) INTO v_temp_live_count; + EXECUTE format(' + SELECT count(*) FROM %1$I tmp JOIN external_users live + ON tmp.aliases && live.aliases AND tmp.id != live.id + ', p_temp_table) INTO v_temp_live_no_del_filter; + EXECUTE format(' + SELECT count(*) FROM %1$I tmp JOIN external_users live + ON tmp.id = live.id + ', p_temp_table) INTO v_temp_live_id_eq; + EXECUTE format(' + SELECT count(*) FROM %1$I tmp + JOIN LATERAL unnest(COALESCE(tmp.aliases, ARRAY[]::text[])) AS t_alias(val) ON true + JOIN external_users live ON live.deleted_at IS NULL AND live.id <> tmp.id + WHERE t_alias.val = ANY(COALESCE(live.aliases, ARRAY[]::text[])) + ', p_temp_table) INTO v_temp_live_null_aliases; + PERFORM _debug_log('step1_candidates', jsonb_build_object( + 'temp_live_count_using_overlap_op', v_temp_live_count, + 'temp_live_count_without_deleted_filter', v_temp_live_no_del_filter, + 'temp_live_count_with_id_eq', v_temp_live_id_eq, + 'temp_live_count_via_unnest_any', v_temp_live_null_aliases + )); + END IF; + IF NOT EXISTS (SELECT 1 FROM _eu_edges) THEN + IF v_debug THEN + PERFORM _debug_log('short_circuit_entry', jsonb_build_object('reason', 'empty _eu_edges')); + + -- Post-hoc: find any temp↔live pair that overlaps via unnest + = ANY. + -- If this returns rows, the && operator missed them and the short- + -- circuit branch will fail with a partial unique index violation. + EXECUTE format(' + SELECT jsonb_agg(row_to_json(r)) + FROM ( + SELECT tmp.id AS tmp_id, tmp.aliases AS tmp_aliases, + live.id AS live_id, live.aliases AS live_aliases, + t_alias.val AS shared_alias + FROM %1$I tmp + JOIN LATERAL unnest(COALESCE(tmp.aliases, ARRAY[]::text[])) AS t_alias(val) ON true + JOIN external_users live + ON live.deleted_at IS NULL + AND live.id <> tmp.id + AND t_alias.val = ANY(COALESCE(live.aliases, ARRAY[]::text[])) + LIMIT 20 + ) r + ', p_temp_table) INTO v_sample; + PERFORM _debug_log('short_circuit_collisions_via_unnest', v_sample); + END IF; + EXECUTE format(' INSERT INTO external_users (id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by) SELECT id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by FROM %I @@ -60,6 +184,33 @@ BEGIN INSERT INTO _eu_merges (loser_id, winner_id) SELECT node, leader FROM _eu_comp WHERE node != leader; + IF v_debug THEN + PERFORM _debug_log('step3_merges', jsonb_build_object( + 'comp_count', (SELECT count(*) FROM _eu_comp), + 'merges_count', (SELECT count(*) FROM _eu_merges), + 'sample_merges', (SELECT jsonb_agg(jsonb_build_object('loser', m.loser_id, 'winner', m.winner_id)) + FROM (SELECT * FROM _eu_merges LIMIT 20) m) + )); + END IF; + + -- Step 3a: Pre-soft-delete any live losers BEFORE Step 3b inserts the temp + -- winners. Without this, the partial unique index on aliases still contains + -- the live loser row at the moment the temp winner is inserted, and the + -- INSERT fires `external_users_aliases_key` whenever the two rows share an + -- alias set (common: one was produced from the same scraper source with a + -- different hash, and the rows carry the same email/descriptor aliases). + -- The losers are moved out of the `WHERE deleted_at IS NULL` partial index + -- here; their alias union is still merged into the winners later in Step 5. + UPDATE external_users SET deleted_at = NOW() + FROM _eu_merges mp + WHERE external_users.id = mp.loser_id + AND external_users.deleted_at IS NULL; + + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step3a_presoft_delete_live_losers', jsonb_build_object('rows_affected', v_row_count)); + END IF; + -- Step 3b: Pre-insert winners from temp table so FK remaps don't violate constraints EXECUTE format(' INSERT INTO external_users (id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by) @@ -68,6 +219,11 @@ BEGIN ON CONFLICT (id) DO NOTHING ', p_temp_table); + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step3b_preinsert', jsonb_build_object('rows_affected', v_row_count)); + END IF; + -- Step 4: Remap FKs in bulk without violating unique constraints CREATE TEMP TABLE _eu_ca_dups (id TEXT PRIMARY KEY) ON COMMIT DROP; INSERT INTO _eu_ca_dups (id) @@ -172,6 +328,11 @@ BEGIN EXECUTE format('DELETE FROM %I USING _eu_merges mp WHERE id = mp.loser_id', p_temp_table); + IF v_debug THEN + EXECUTE format('SELECT count(*) FROM %I', p_temp_table) INTO v_row_count; + PERFORM _debug_log('step7_post_delete_temp', jsonb_build_object('remaining_temp_rows', v_row_count)); + END IF; + -- Step 8: Upsert survivors EXECUTE format(' INSERT INTO external_users (id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by) @@ -183,6 +344,11 @@ BEGIN updated_at = EXCLUDED.updated_at, deleted_at = NULL ', p_temp_table); + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step8_upsert', jsonb_build_object('rows_affected', v_row_count)); + END IF; + RETURN QUERY SELECT mp.loser_id, mp.winner_id FROM _eu_merges mp; END; $$ LANGUAGE plpgsql; @@ -194,6 +360,17 @@ BEGIN LOCK TABLE config_access, access_reviews, config_access_logs, external_user_groups, external_users, external_groups, external_roles IN SHARE ROW EXCLUSIVE MODE; + -- Step 0: Normalize temp aliases to match the normalize_aliases() trigger + -- on external_groups (lowercase + sorted + distinct). The edge-build below + -- uses byte-exact && which would otherwise miss any temp row whose aliases + -- differ from the normalized live row only by case. See the equivalent + -- Step 0 in merge_and_upsert_external_users for details. + EXECUTE format(' + UPDATE %1$I SET + aliases = ARRAY(SELECT DISTINCT LOWER(elem) FROM unnest(aliases) AS elem ORDER BY LOWER(elem)) + WHERE aliases IS NOT NULL + ', p_temp_table); + EXECUTE format(' CREATE TEMP TABLE _eg_edges ON COMMIT DROP AS SELECT DISTINCT a.id AS id1, b.id AS id2 @@ -240,6 +417,15 @@ BEGIN CREATE TEMP TABLE _eg_merges (loser_id UUID PRIMARY KEY, winner_id UUID) ON COMMIT DROP; INSERT INTO _eg_merges SELECT node, leader FROM _eg_comp WHERE node != leader; + -- Step 3a: Pre-soft-delete live losers BEFORE pre-insert so the partial + -- unique index doesn't fire on a winner with identical aliases to a + -- still-active loser. See the equivalent Step 3a in + -- merge_and_upsert_external_users for details. + UPDATE external_groups SET deleted_at = NOW() + FROM _eg_merges mp + WHERE external_groups.id = mp.loser_id + AND external_groups.deleted_at IS NULL; + EXECUTE format(' INSERT INTO external_groups (id, aliases, name, account_id, scraper_id, group_type, created_at, updated_at) SELECT id, aliases, name, account_id, scraper_id, group_type, created_at, updated_at @@ -339,6 +525,14 @@ BEGIN LOCK TABLE config_access, access_reviews, config_access_logs, external_user_groups, external_users, external_groups, external_roles IN SHARE ROW EXCLUSIVE MODE; + -- Step 0: Normalize temp aliases to match the normalize_aliases() trigger + -- on external_roles. See Step 0 in merge_and_upsert_external_users. + EXECUTE format(' + UPDATE %1$I SET + aliases = ARRAY(SELECT DISTINCT LOWER(elem) FROM unnest(aliases) AS elem ORDER BY LOWER(elem)) + WHERE aliases IS NOT NULL + ', p_temp_table); + EXECUTE format(' CREATE TEMP TABLE _er_edges ON COMMIT DROP AS SELECT DISTINCT a.id AS id1, b.id AS id2 @@ -386,6 +580,13 @@ BEGIN CREATE TEMP TABLE _er_merges (loser_id UUID PRIMARY KEY, winner_id UUID) ON COMMIT DROP; INSERT INTO _er_merges SELECT node, leader FROM _er_comp WHERE node != leader; + -- Step 3a: Pre-soft-delete live losers BEFORE pre-insert. See Step 3a in + -- merge_and_upsert_external_users. + UPDATE external_roles SET deleted_at = NOW() + FROM _er_merges mp + WHERE external_roles.id = mp.loser_id + AND external_roles.deleted_at IS NULL; + EXECUTE format(' INSERT INTO external_roles (id, aliases, name, account_id, role_type, description, scraper_id, application_id, created_at, updated_at) SELECT id, aliases, name, account_id, role_type, description, scraper_id, application_id, created_at, updated_at From fbc428ec770ecdc2886c68fd8c3c89cae3362275 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 14:47:05 +0300 Subject: [PATCH 11/12] fix(db): ensure loser ids remain discoverable via aliases after entity merges Include loser ids in the alias union when merging entities so that future lookups by the old id can recover the winner. Applies to users, groups, and roles. Also refactors alias merging to use parametrized queries and left-joins both temp and live tables. --- views/045_merge_external_entities.sql | 112 +++++++++++++++++++------- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/views/045_merge_external_entities.sql b/views/045_merge_external_entities.sql index 0cc42af19..a2e03155e 100644 --- a/views/045_merge_external_entities.sql +++ b/views/045_merge_external_entities.sql @@ -297,22 +297,40 @@ BEGIN DELETE FROM external_user_groups USING _eu_merges mp WHERE external_user_groups.external_user_id = mp.loser_id; - -- Step 5: Merge aliases from losers into winners in live table - UPDATE external_users SET - aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_users.aliases || agg.all_aliases) ORDER BY 1), '{}'::text[]) - FROM ( - SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases - FROM _eu_merges mp JOIN external_users eu ON eu.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(eu.aliases, '{}'::text[])) AS a - GROUP BY mp.winner_id - ) agg - WHERE external_users.id = agg.winner_id; + -- Step 5: Merge aliases from losers into winners in the live table. + -- We also append `loser.id::text` to the winner's alias union so that + -- future lookups by the loser's old id can recover the winner. + -- entity.aliases never contains entity.id for live rows; loser ids only + -- become aliases here, at the moment they stop being canonical. + -- + -- A loser may live in the temp table (LEFT JOIN tmp), the live table + -- (LEFT JOIN external_users) or both — we union all available alias + -- sources together with the loser id itself. + EXECUTE format(' + UPDATE external_users SET + aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_users.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) + FROM ( + SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases + FROM _eu_merges mp + LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id + LEFT JOIN external_users live_src ON live_src.id = mp.loser_id + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a + GROUP BY mp.winner_id + ) agg + WHERE external_users.id = agg.winner_id + ', p_temp_table); -- Step 6: Soft-delete losers UPDATE external_users SET deleted_at = NOW() FROM _eu_merges mp WHERE external_users.id = mp.loser_id; - -- Step 7: Consolidate temp table (merge aliases from all losers - both temp and live - into temp winners) + -- Step 7: Consolidate temp table (merge aliases from all losers - both + -- temp and live - into temp winners). As in Step 5, also append the + -- loser id itself so it remains lookupable via aliases after the merge. EXECUTE format(' UPDATE %1$I t SET aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(t.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) @@ -321,7 +339,11 @@ BEGIN FROM _eu_merges mp LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id LEFT JOIN external_users live_src ON live_src.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(tmp_src.aliases, ''{}''::text[]) || COALESCE(live_src.aliases, ''{}''::text[])) AS a + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a GROUP BY mp.winner_id ) agg WHERE t.id = agg.winner_id ', p_temp_table); @@ -478,14 +500,25 @@ BEGIN DELETE FROM external_user_groups USING _eg_merges mp WHERE external_user_groups.external_group_id = mp.loser_id; - UPDATE external_groups SET - aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_groups.aliases || agg.all_aliases) ORDER BY 1), '{}'::text[]) - FROM ( - SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases - FROM _eg_merges mp JOIN external_groups eg ON eg.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(eg.aliases, '{}'::text[])) AS a - GROUP BY mp.winner_id - ) agg WHERE external_groups.id = agg.winner_id; + -- Merge loser aliases (and the loser id itself) into the winner. See the + -- equivalent step in merge_and_upsert_external_users for the reasoning, + -- including why we LEFT JOIN both temp and live as alias sources. + EXECUTE format(' + UPDATE external_groups SET + aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_groups.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) + FROM ( + SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases + FROM _eg_merges mp + LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id + LEFT JOIN external_groups live_src ON live_src.id = mp.loser_id + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a + GROUP BY mp.winner_id + ) agg WHERE external_groups.id = agg.winner_id + ', p_temp_table); UPDATE external_groups SET deleted_at = NOW() FROM _eg_merges mp WHERE external_groups.id = mp.loser_id; @@ -498,7 +531,11 @@ BEGIN FROM _eg_merges mp LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id LEFT JOIN external_groups live_src ON live_src.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(tmp_src.aliases, ''{}''::text[]) || COALESCE(live_src.aliases, ''{}''::text[])) AS a + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a GROUP BY mp.winner_id ) agg WHERE t.id = agg.winner_id ', p_temp_table); @@ -633,14 +670,25 @@ BEGIN UPDATE access_reviews SET external_role_id = mp.winner_id FROM _er_merges mp WHERE access_reviews.external_role_id = mp.loser_id; - UPDATE external_roles SET - aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_roles.aliases || agg.all_aliases) ORDER BY 1), '{}'::text[]) - FROM ( - SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases - FROM _er_merges mp JOIN external_roles er ON er.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(er.aliases, '{}'::text[])) AS a - GROUP BY mp.winner_id - ) agg WHERE external_roles.id = agg.winner_id; + -- Merge loser aliases (and the loser id itself) into the winner. See the + -- equivalent step in merge_and_upsert_external_users for the reasoning, + -- including why we LEFT JOIN both temp and live as alias sources. + EXECUTE format(' + UPDATE external_roles SET + aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_roles.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) + FROM ( + SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases + FROM _er_merges mp + LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id + LEFT JOIN external_roles live_src ON live_src.id = mp.loser_id + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a + GROUP BY mp.winner_id + ) agg WHERE external_roles.id = agg.winner_id + ', p_temp_table); UPDATE external_roles SET deleted_at = NOW() FROM _er_merges mp WHERE external_roles.id = mp.loser_id; @@ -653,7 +701,11 @@ BEGIN FROM _er_merges mp LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id LEFT JOIN external_roles live_src ON live_src.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(tmp_src.aliases, ''{}''::text[]) || COALESCE(live_src.aliases, ''{}''::text[])) AS a + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a GROUP BY mp.winner_id ) agg WHERE t.id = agg.winner_id ', p_temp_table); From ff4d322331cadacc754828013657ccc7fd526fda Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Mon, 13 Apr 2026 18:06:54 +0300 Subject: [PATCH 12/12] feat(changegroup): add change grouping engine and models Implements a rule-based engine to automatically group correlated config_changes into logical change_groups. Supports time-windowed grouping (pod startups), fan-out detection (deployments), temporary access tracking, and incident correlation. Includes explicit group creation and optional periodic closure based on inactivity windows. --- changegroup/closer.go | 108 +++++++ changegroup/engine.go | 175 ++++++++++++ changegroup/errors.go | 41 +++ changegroup/explicit.go | 80 ++++++ changegroup/merge.go | 183 ++++++++++++ changegroup/merge_test.go | 143 ++++++++++ changegroup/pseudo.go | 118 ++++++++ changegroup/pseudo_test.go | 91 ++++++ changegroup/rule.go | 191 +++++++++++++ changegroup/upsert.go | 197 +++++++++++++ models/change_group.go | 50 ++++ models/changes.go | 1 + models/config.go | 8 + query/change_groups.go | 150 ++++++++++ query/config_changes.go | 27 ++ rbac/objects.go | 2 + schema/config.hcl | 118 ++++++++ tests/change_groups_test.go | 435 +++++++++++++++++++++++++++++ types/config_change_groups.go | 171 ++++++++++++ types/config_change_groups_test.go | 117 ++++++++ views/030_config_changes.sql | 3 +- views/047_change_groups.sql | 69 +++++ 22 files changed, 2477 insertions(+), 1 deletion(-) create mode 100644 changegroup/closer.go create mode 100644 changegroup/engine.go create mode 100644 changegroup/errors.go create mode 100644 changegroup/explicit.go create mode 100644 changegroup/merge.go create mode 100644 changegroup/merge_test.go create mode 100644 changegroup/pseudo.go create mode 100644 changegroup/pseudo_test.go create mode 100644 changegroup/rule.go create mode 100644 changegroup/upsert.go create mode 100644 models/change_group.go create mode 100644 query/change_groups.go create mode 100644 tests/change_groups_test.go create mode 100644 types/config_change_groups.go create mode 100644 types/config_change_groups_test.go create mode 100644 views/047_change_groups.sql diff --git a/changegroup/closer.go b/changegroup/closer.go new file mode 100644 index 000000000..e3564628c --- /dev/null +++ b/changegroup/closer.go @@ -0,0 +1,108 @@ +package changegroup + +import ( + "encoding/json" + "time" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" +) + +// CloseStaleGroups closes open change_groups whose last_member_at is older +// than the effective close window for the group's rule. Called periodically +// by StartCloser. +// +// For TemporaryPermissionGroup specifically, this also computes +// DurationSeconds from started_at/ended_at at close time. +func (e *Engine) CloseStaleGroups(ctx context.Context, now time.Time) (int, error) { + // Build the rule → CloseAfter lookup snapshot. + e.mu.RLock() + closeAfterByRule := make(map[string]time.Duration, len(e.rules)) + for _, r := range e.rules { + ca := r.CloseAfter.Std() + if ca == 0 { + // 0 means "never time out" — skip timeout-close for this rule. + continue + } + closeAfterByRule[r.Name] = ca + } + e.mu.RUnlock() + + if len(closeAfterByRule) == 0 { + return 0, nil + } + + var candidates []models.ChangeGroup + if err := ctx.DB(). + Where("status = ?", models.ChangeGroupStatusOpen). + Find(&candidates).Error; err != nil { + return 0, err + } + + closed := 0 + for i := range candidates { + g := &candidates[i] + if g.RuleName == nil { + continue + } + window, ok := closeAfterByRule[*g.RuleName] + if !ok { + continue + } + if now.Sub(g.LastMemberAt) < window { + continue + } + if err := finalizeClose(ctx, g, g.LastMemberAt); err != nil { + return closed, err + } + closed++ + } + return closed, nil +} + +// finalizeClose writes the terminal state for a group: status=closed, +// ended_at set, and — for TemporaryPermissionGroup — DurationSeconds computed. +func finalizeClose(ctx context.Context, g *models.ChangeGroup, endedAt time.Time) error { + updates := map[string]any{ + "status": models.ChangeGroupStatusClosed, + "ended_at": endedAt, + "updated_at": time.Now().UTC(), + } + + stored, err := g.TypedDetails() + if err == nil && stored != nil { + if tp, ok := stored.(types.TemporaryPermissionGroup); ok { + dur := int64(endedAt.Sub(g.StartedAt).Seconds()) + tp.DurationSeconds = &dur + raw, err := json.Marshal(tp) + if err == nil { + updates["details"] = types.JSON(raw) + } + } + } + + return ctx.DB().Model(&models.ChangeGroup{}). + Where("id = ?", g.ID). + Updates(updates).Error +} + +// StartCloser runs CloseStaleGroups on a ticker until ctx is cancelled. +// Intended to be spawned once per process by the job scheduler. +func (e *Engine) StartCloser(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = 30 * time.Second + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-ticker.C: + _, _ = e.CloseStaleGroups(ctx, now) + } + } + }() +} diff --git a/changegroup/engine.go b/changegroup/engine.go new file mode 100644 index 000000000..5d970ac26 --- /dev/null +++ b/changegroup/engine.go @@ -0,0 +1,175 @@ +package changegroup + +import ( + "sort" + "sync" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" +) + +// Engine evaluates GroupingRules against incoming config_changes and keeps +// change_groups in sync. It is safe for concurrent use once rules are loaded. +type Engine struct { + mu sync.RWMutex + rules []*GroupingRule + evaluator Evaluator +} + +// New returns an Engine pre-loaded with the given rules and evaluator. +// The caller is responsible for calling Validate on each rule before passing +// it in — New re-validates and returns an error if any rule fails. +func New(evaluator Evaluator, rules []GroupingRule) (*Engine, error) { + e := &Engine{evaluator: evaluator} + if err := e.SetRules(rules); err != nil { + return nil, err + } + return e, nil +} + +// SetRules replaces the engine's rule set atomically. All rules are +// (re-)validated; if any rule fails, the previous rule set is preserved and +// the error is returned. +func (e *Engine) SetRules(rules []GroupingRule) error { + if e.evaluator == nil { + // Allow rule loading with no evaluator only if every rule has empty + // CEL expressions (currently never true). Require evaluator. + return ErrMissingEvaluator + } + + compiled := make([]*GroupingRule, 0, len(rules)) + for i := range rules { + r := rules[i] + if err := r.Validate(e.evaluator); err != nil { + return err + } + compiled = append(compiled, &r) + } + sort.SliceStable(compiled, func(i, j int) bool { + return compiled[i].Priority > compiled[j].Priority + }) + + e.mu.Lock() + e.rules = compiled + e.mu.Unlock() + return nil +} + +// Rules returns a snapshot of the currently loaded rules for inspection/tests. +func (e *Engine) Rules() []*GroupingRule { + e.mu.RLock() + defer e.mu.RUnlock() + out := make([]*GroupingRule, len(e.rules)) + copy(out, e.rules) + return out +} + +// Evaluate runs the rule engine against a single already-persisted +// config_changes row. Matching rules will create or update a change_group and +// set change.GroupID. If the change already has a GroupID, Evaluate is a no-op +// (producers are trusted). +func (e *Engine) Evaluate(ctx context.Context, change *models.ConfigChange) error { + if change == nil { + return nil + } + if change.GroupID != nil { + return nil // explicit path — respect producer assignment + } + + e.mu.RLock() + rules := e.rules + e.mu.RUnlock() + + for _, rule := range rules { + matched, err := e.tryRule(ctx, rule, change) + if err != nil { + return err + } + if matched { + return nil + } + } + return nil +} + +// tryRule attempts to apply a single rule to the change. Returns (true, nil) +// on a successful attach, (false, nil) on a non-match. +func (e *Engine) tryRule(ctx context.Context, rule *GroupingRule, change *models.ConfigChange) (bool, error) { + if !rule.Matches(change.ChangeType) { + return false, nil + } + + env := buildSingleChangeEnv(change) + + if rule.filterProgram != nil { + ok, err := e.evaluator.EvalBool(rule.filterProgram, env) + if err != nil { + return false, &EvalError{Rule: rule.Name, Field: "filter", Err: err} + } + if !ok { + return false, nil + } + } + + rawKey, err := e.evaluator.EvalString(rule.keyProgram, env) + if err != nil { + return false, &EvalError{Rule: rule.Name, Field: "key", Err: err} + } + if rawKey == "" { + return false, nil + } + correlationKey := hashKey(rule.Name, rawKey) + + if err := e.upsertAndAttach(ctx, rule, correlationKey, change); err != nil { + return false, err + } + return true, nil +} + +// buildSingleChangeEnv creates an Env whose Changes contains only the +// triggering change (plus the flat shortcuts). Used on the first call; the +// upsert path rebuilds the env with all persisted members before re-running +// Details / Summary. +func buildSingleChangeEnv(change *models.ConfigChange) Env { + m := changeAsMap(change) + return Env{ + Change: m, + Changes: []map[string]any{m}, + Flat: m, + } +} + +// changeAsMap projects a ConfigChange into the CEL binding shape. +func changeAsMap(c *models.ConfigChange) map[string]any { + var groupID any + if c.GroupID != nil { + groupID = c.GroupID.String() + } + return map[string]any{ + "id": c.ID, + "external_id": c.ExternalID, + "external_change_id": derefString(c.ExternalChangeID), + "config_id": c.ConfigID, + "change_type": c.ChangeType, + "severity": string(c.Severity), + "source": c.Source, + "summary": c.Summary, + "patches": c.Patches, + "diff": c.Diff, + "fingerprint": c.Fingerprint, + "details": c.Details, + "created_at": c.CreatedAt, + "created_by": c.CreatedBy, + "external_created_by": derefString(c.ExternalCreatedBy), + "count": c.Count, + "group_id": groupID, + } +} + +func derefString(s *string) string { + if s == nil { + return "" + } + return *s +} + diff --git a/changegroup/errors.go b/changegroup/errors.go new file mode 100644 index 000000000..b2ad170a9 --- /dev/null +++ b/changegroup/errors.go @@ -0,0 +1,41 @@ +package changegroup + +import ( + "errors" + "fmt" +) + +var ( + ErrEmptyRuleName = errors.New("changegroup: rule name is required") + ErrMissingDetails = errors.New("changegroup: rule details expression is required") + ErrMissingKey = errors.New("changegroup: rule key expression is required") + ErrUnknownPseudo = errors.New("changegroup: unknown pseudo change type") + ErrMissingEvaluator = errors.New("changegroup: evaluator must be set before rules are loaded") +) + +// EvalError wraps an evaluator runtime failure with the originating rule and field. +type EvalError struct { + Rule string + Field string + Err error +} + +func (e *EvalError) Error() string { + return fmt.Sprintf("changegroup: eval rule %q field %q: %v", e.Rule, e.Field, e.Err) +} + +func (e *EvalError) Unwrap() error { return e.Err } + +// CompileError wraps an evaluator compile failure with the originating rule +// and field so operators can pinpoint their YAML. +type CompileError struct { + Rule string + Field string + Err error +} + +func (e *CompileError) Error() string { + return fmt.Sprintf("changegroup: compile rule %q field %q: %v", e.Rule, e.Field, e.Err) +} + +func (e *CompileError) Unwrap() error { return e.Err } diff --git a/changegroup/explicit.go b/changegroup/explicit.go new file mode 100644 index 000000000..dff1f52fa --- /dev/null +++ b/changegroup/explicit.go @@ -0,0 +1,80 @@ +package changegroup + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" +) + +// Create inserts a new change_group with an explicit source. Producers call +// this when they know upfront which changes belong together (e.g. a playbook +// emits a coordinated deployment across several configs). +// +// The correlation key defaults to the group id if empty — explicit groups are +// keyed on identity, not content. +func Create(ctx context.Context, group models.ChangeGroup) (uuid.UUID, error) { + if group.ID == uuid.Nil { + group.ID = uuid.New() + } + if group.CorrelationKey == "" { + group.CorrelationKey = "explicit:" + group.ID.String() + } + if group.Source == "" { + group.Source = models.ChangeGroupSourceExplicit + } + if group.Status == "" { + group.Status = models.ChangeGroupStatusOpen + } + now := time.Now().UTC() + if group.StartedAt.IsZero() { + group.StartedAt = now + } + if group.LastMemberAt.IsZero() { + group.LastMemberAt = group.StartedAt + } + if group.CreatedAt.IsZero() { + group.CreatedAt = now + } + if group.UpdatedAt.IsZero() { + group.UpdatedAt = now + } + + if err := ctx.DB().Create(&group).Error; err != nil { + return uuid.Nil, err + } + return group.ID, nil +} + +// CreateTyped is a convenience that builds a ChangeGroup from a typed +// GroupType details value plus the usual metadata fields. +func CreateTyped( + ctx context.Context, + kind types.GroupType, + summary string, +) (uuid.UUID, error) { + raw, err := json.Marshal(kind) + if err != nil { + return uuid.Nil, err + } + return Create(ctx, models.ChangeGroup{ + Type: kind.Kind(), + Summary: summary, + Details: types.JSON(raw), + }) +} + +// Assign attaches the given already-persisted config_changes rows to the +// given group. The 047 trigger maintains member_count and time bounds. +func Assign(ctx context.Context, groupID uuid.UUID, changeIDs ...string) error { + if len(changeIDs) == 0 { + return nil + } + return ctx.DB().Model(&models.ConfigChange{}). + Where("id IN ?", changeIDs). + Update("group_id", groupID).Error +} diff --git a/changegroup/merge.go b/changegroup/merge.go new file mode 100644 index 000000000..7503be050 --- /dev/null +++ b/changegroup/merge.go @@ -0,0 +1,183 @@ +package changegroup + +import ( + "fmt" + "reflect" + "time" + + "github.com/flanksource/duty/types" +) + +// Merge combines new typed group details into the existing stored value, +// using `merge:"..."` struct tags on the concrete GroupType to decide per-field +// strategy. Both arguments must be non-nil and of the same concrete type. +// +// Strategies: +// - append: dedupe-append slices. +// - firstSet: keep the existing value if it was already set; otherwise take new. +// - min / max: reduce on comparable scalars (time.Time, numbers, strings). +// - (default): last-write-wins for scalars; new slice/map replaces existing +// iff the new value is non-empty, otherwise the old value is preserved. +// +// Nil pointers / zero slices on the "new" side never clobber a populated +// existing value — this lets CEL omit fields a given member doesn't know about. +func Merge(existing, incoming types.GroupType) (types.GroupType, error) { + if existing == nil { + return incoming, nil + } + if incoming == nil { + return existing, nil + } + if existing.Kind() != incoming.Kind() { + return nil, fmt.Errorf("changegroup: cannot merge %q with %q", existing.Kind(), incoming.Kind()) + } + + // Work on addressable copies so reflect.Set works even when callers pass + // value types (e.g. types.DeploymentGroup{}). + ex := reflect.New(reflect.TypeOf(existing)).Elem() + ex.Set(reflect.ValueOf(existing)) + in := reflect.ValueOf(incoming) + + if ex.Kind() != reflect.Struct { + return nil, fmt.Errorf("changegroup: cannot merge non-struct GroupType %q", existing.Kind()) + } + + for i := 0; i < ex.NumField(); i++ { + field := ex.Type().Field(i) + if !field.IsExported() { + continue + } + strategy := field.Tag.Get("merge") + exField := ex.Field(i) + inField := in.Field(i) + mergeField(exField, inField, strategy) + } + + return ex.Interface().(types.GroupType), nil +} + +func mergeField(dst, src reflect.Value, strategy string) { + // Never clobber with an explicit zero value on the incoming side — this + // lets rules omit fields they don't care about. + if isZero(src) { + return + } + + switch strategy { + case "append": + mergeAppend(dst, src) + case "firstSet": + if isZero(dst) { + dst.Set(src) + } + case "min": + if isZero(dst) || lessThan(src, dst) { + dst.Set(src) + } + case "max": + if isZero(dst) || lessThan(dst, src) { + dst.Set(src) + } + case "mapMerge": + mergeMap(dst, src) + default: + // last-write-wins for scalars; replace for slices/maps when src is non-empty. + dst.Set(src) + } +} + +// mergeMap performs a per-key merge of two maps of the same type. Keys only +// in dst are preserved; keys in both take the src value (last-write-wins). +func mergeMap(dst, src reflect.Value) { + if dst.Kind() != reflect.Map || src.Kind() != reflect.Map { + dst.Set(src) + return + } + if dst.IsNil() { + dst.Set(reflect.MakeMapWithSize(dst.Type(), src.Len())) + } + iter := src.MapRange() + for iter.Next() { + dst.SetMapIndex(iter.Key(), iter.Value()) + } +} + +// mergeAppend performs a dedupe-append of src slice elements into dst. +// Both must be slices of the same element kind. +func mergeAppend(dst, src reflect.Value) { + if dst.Kind() != reflect.Slice || src.Kind() != reflect.Slice { + // Mis-tagged field; fall back to replace. + dst.Set(src) + return + } + seen := make(map[any]struct{}, dst.Len()+src.Len()) + result := reflect.MakeSlice(dst.Type(), 0, dst.Len()+src.Len()) + for i := 0; i < dst.Len(); i++ { + v := dst.Index(i).Interface() + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + result = reflect.Append(result, dst.Index(i)) + } + for i := 0; i < src.Len(); i++ { + v := src.Index(i).Interface() + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + result = reflect.Append(result, src.Index(i)) + } + dst.Set(result) +} + +// isZero reports whether v holds its type's zero value. For pointers, a nil +// pointer is zero; for slices/maps, len == 0 is zero. +func isZero(v reflect.Value) bool { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + return v.IsNil() + case reflect.Slice, reflect.Map: + return v.Len() == 0 + default: + return v.IsZero() + } +} + +// lessThan compares two values of the same comparable type. It supports +// time.Time, signed/unsigned integers, floats and strings. For any other +// type it returns false (conservative: don't overwrite). +func lessThan(a, b reflect.Value) bool { + // Dereference pointers. + for a.Kind() == reflect.Ptr { + if a.IsNil() { + return true + } + a = a.Elem() + } + for b.Kind() == reflect.Ptr { + if b.IsNil() { + return false + } + b = b.Elem() + } + + if t, ok := a.Interface().(time.Time); ok { + if u, ok := b.Interface().(time.Time); ok { + return t.Before(u) + } + } + + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return a.Uint() < b.Uint() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.String: + return a.String() < b.String() + } + return false +} + diff --git a/changegroup/merge_test.go b/changegroup/merge_test.go new file mode 100644 index 000000000..2c382797a --- /dev/null +++ b/changegroup/merge_test.go @@ -0,0 +1,143 @@ +package changegroup + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/onsi/gomega" + + "github.com/flanksource/duty/types" +) + +func TestMergeAppend(t *testing.T) { + g := gomega.NewWithT(t) + + a := uuid.New() + b := uuid.New() + c := uuid.New() + + existing := types.DeploymentGroup{ + Image: "registry/app:v1", + TargetConfigIDs: []uuid.UUID{a, b}, + } + incoming := types.DeploymentGroup{ + Image: "registry/app:v1", + TargetConfigIDs: []uuid.UUID{b, c}, // b is a duplicate + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + dep := out.(types.DeploymentGroup) + g.Expect(dep.TargetConfigIDs).To(gomega.Equal([]uuid.UUID{a, b, c})) + g.Expect(dep.Image).To(gomega.Equal("registry/app:v1")) +} + +func TestMergeFirstSet(t *testing.T) { + g := gomega.NewWithT(t) + + grant := uuid.New() + revoke := uuid.New() + otherGrant := uuid.New() + + existing := types.TemporaryPermissionGroup{ + UserID: "u1", + GrantChangeID: &grant, + } + // "incoming" arrives with a new revoke and a (wrong) new grant id — firstSet + // must keep the old grant id. + incoming := types.TemporaryPermissionGroup{ + UserID: "u1", + GrantChangeID: &otherGrant, + RevokeChangeID: &revoke, + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + tp := out.(types.TemporaryPermissionGroup) + g.Expect(tp.GrantChangeID).To(gomega.Equal(&grant)) + g.Expect(tp.RevokeChangeID).To(gomega.Equal(&revoke)) +} + +func TestMergeMinMax(t *testing.T) { + g := gomega.NewWithT(t) + + t1 := time.Date(2026, 1, 1, 10, 0, 0, 0, time.UTC) + t2 := time.Date(2026, 1, 1, 11, 0, 0, 0, time.UTC) + t3 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + + existing := types.IncidentResponseGroup{ + IncidentID: "INC-1", + OpenedAt: t2, + ClosedAt: t2, + PlaybookRunIDs: []uuid.UUID{uuid.New()}, + } + incoming := types.IncidentResponseGroup{ + IncidentID: "INC-1", + OpenedAt: t1, // earlier → wins min + ClosedAt: t3, // later → wins max + PlaybookRunIDs: []uuid.UUID{uuid.New()}, + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + ir := out.(types.IncidentResponseGroup) + g.Expect(ir.OpenedAt).To(gomega.Equal(t1)) + g.Expect(ir.ClosedAt).To(gomega.Equal(t3)) + g.Expect(ir.PlaybookRunIDs).To(gomega.HaveLen(2)) +} + +func TestMergeIgnoresZeroIncoming(t *testing.T) { + g := gomega.NewWithT(t) + + existing := types.DeploymentGroup{ + Image: "registry/app:v1", + Version: "v1", + } + // Incoming omits Version — existing Version must survive. + incoming := types.DeploymentGroup{ + Image: "registry/app:v2", + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + dep := out.(types.DeploymentGroup) + g.Expect(dep.Image).To(gomega.Equal("registry/app:v2"), "scalar last-write-wins") + g.Expect(dep.Version).To(gomega.Equal("v1"), "zero incoming must not clobber") +} + +func TestMergeKindMismatch(t *testing.T) { + g := gomega.NewWithT(t) + + _, err := Merge( + types.DeploymentGroup{Image: "a"}, + types.PromotionGroup{Version: "v1"}, + ) + g.Expect(err).To(gomega.HaveOccurred()) +} + +func TestMergeCustomGroup(t *testing.T) { + g := gomega.NewWithT(t) + + existing := types.CustomGroup{Fields: map[string]any{"a": 1, "b": 2}} + incoming := types.CustomGroup{Fields: map[string]any{"b": 20, "c": 3}} + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + cg := out.(types.CustomGroup) + g.Expect(cg.Fields).To(gomega.Equal(map[string]any{"a": 1, "b": 20, "c": 3})) +} + +func TestMergeNilFirstMember(t *testing.T) { + g := gomega.NewWithT(t) + + incoming := types.DeploymentGroup{Image: "a"} + out, err := Merge(nil, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(out).To(gomega.Equal(incoming)) +} diff --git a/changegroup/pseudo.go b/changegroup/pseudo.go new file mode 100644 index 000000000..f910c1e94 --- /dev/null +++ b/changegroup/pseudo.go @@ -0,0 +1,118 @@ +package changegroup + +import ( + "fmt" + "sort" + + "github.com/flanksource/duty/types" +) + +// Pseudo change-type identifiers used in rule DSL. Entries in +// GroupingRule.ChangeTypes starting with "@" are expanded via ExpandPseudo. +const ( + PseudoCreated = "@created" + PseudoChanged = "@changed" + PseudoDeleted = "@deleted" + PseudoHealthy = "@healthy" + PseudoUnhealthy = "@unhealthy" + PseudoPlaybook = "@playbook" +) + +// pseudoMap defines the literal change_type set for each pseudo identifier. +// Keyed on the pseudo string (leading "@" preserved). Kept private to this +// file; callers use ExpandPseudo. +var pseudoMap = map[string]map[string]struct{}{ + PseudoCreated: setOf( + types.ChangeTypeCreate, + "Created", + types.ChangeTypeUserCreated, + types.ChangeTypeRegisterNode, + types.ChangeTypeRunInstances, + ), + PseudoDeleted: setOf( + types.ChangeTypeDelete, + "Deleted", + types.ChangeTypeUserDeleted, + ), + PseudoHealthy: setOf( + "Healthy", + types.ChangeTypeBackupCompleted, + types.ChangeTypeBackupRestored, + types.ChangeTypePipelineRunCompleted, + types.ChangeTypePlaybookCompleted, + types.ChangeTypeCertificateRenewed, + ), + PseudoUnhealthy: setOf( + "Unhealthy", + types.ChangeTypeBackupFailed, + types.ChangeTypePipelineRunFailed, + types.ChangeTypePlaybookFailed, + types.ChangeTypeCertificateExpired, + ), + PseudoChanged: setOf( + types.ChangeTypeUpdate, + types.ChangeTypeDiff, + types.ChangeTypeDeployment, + types.ChangeTypePromotion, + types.ChangeTypeRollback, + types.ChangeTypeScaling, + types.ChangeTypeCostChange, + types.ChangeTypePermissionAdded, + types.ChangeTypePermissionRemoved, + types.ChangeTypeGroupMemberAdded, + types.ChangeTypeGroupMemberRemoved, + ), + PseudoPlaybook: setOf( + types.ChangeTypePlaybookStarted, + types.ChangeTypePlaybookCompleted, + types.ChangeTypePlaybookFailed, + ), +} + +func setOf(items ...string) map[string]struct{} { + m := make(map[string]struct{}, len(items)) + for _, it := range items { + m[it] = struct{}{} + } + return m +} + +// ExpandPseudo returns the sorted list of literal change_type strings that +// the given pseudo identifier covers. Returns an error for unknown pseudo +// identifiers. +func ExpandPseudo(pseudo string) ([]string, error) { + set, ok := pseudoMap[pseudo] + if !ok { + return nil, fmt.Errorf("%w: %q", ErrUnknownPseudo, pseudo) + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// expandChangeTypes builds the deduped literal change_type set for a rule's +// ChangeTypes slice. Entries starting with "@" expand via ExpandPseudo; +// everything else is taken as a literal. +func expandChangeTypes(entries []string) (map[string]struct{}, error) { + out := make(map[string]struct{}, len(entries)) + for _, e := range entries { + if e == "" { + continue + } + if e[0] == '@' { + lits, err := ExpandPseudo(e) + if err != nil { + return nil, err + } + for _, l := range lits { + out[l] = struct{}{} + } + continue + } + out[e] = struct{}{} + } + return out, nil +} diff --git a/changegroup/pseudo_test.go b/changegroup/pseudo_test.go new file mode 100644 index 000000000..a0275bac6 --- /dev/null +++ b/changegroup/pseudo_test.go @@ -0,0 +1,91 @@ +package changegroup + +import ( + "errors" + "testing" + + "github.com/onsi/gomega" + + "github.com/flanksource/duty/types" +) + +func TestExpandPseudo(t *testing.T) { + cases := []struct { + pseudo string + want []string + }{ + { + pseudo: PseudoCreated, + want: []string{ + "Created", + types.ChangeTypeCreate, + types.ChangeTypeRegisterNode, + types.ChangeTypeRunInstances, + types.ChangeTypeUserCreated, + }, + }, + { + pseudo: PseudoUnhealthy, + want: []string{ + types.ChangeTypeBackupFailed, + types.ChangeTypeCertificateExpired, + types.ChangeTypePipelineRunFailed, + types.ChangeTypePlaybookFailed, + "Unhealthy", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.pseudo, func(t *testing.T) { + g := gomega.NewWithT(t) + got, err := ExpandPseudo(tc.pseudo) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.ConsistOf(tc.want)) + }) + } +} + +func TestExpandPseudoUnknown(t *testing.T) { + g := gomega.NewWithT(t) + _, err := ExpandPseudo("@nope") + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(errors.Is(err, ErrUnknownPseudo)).To(gomega.BeTrue()) +} + +func TestExpandChangeTypesMixed(t *testing.T) { + g := gomega.NewWithT(t) + + got, err := expandChangeTypes([]string{ + "PermissionAdded", + "@unhealthy", + "PermissionAdded", // dedupe + "", // skip + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + g.Expect(got).To(gomega.HaveKey("PermissionAdded")) + g.Expect(got).To(gomega.HaveKey(types.ChangeTypeBackupFailed)) + g.Expect(got).To(gomega.HaveKey(types.ChangeTypePlaybookFailed)) + g.Expect(got).ToNot(gomega.HaveKey("")) +} + +func TestExpandChangeTypesUnknownPseudoPropagates(t *testing.T) { + g := gomega.NewWithT(t) + + _, err := expandChangeTypes([]string{"Deployment", "@foo"}) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(errors.Is(err, ErrUnknownPseudo)).To(gomega.BeTrue()) +} + +func TestExpandChangeTypesEmptyAcceptsAll(t *testing.T) { + g := gomega.NewWithT(t) + + got, err := expandChangeTypes(nil) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.BeEmpty()) + + // Matches helper treats empty set as "accept everything". + r := &GroupingRule{literalChangeTypes: got} + g.Expect(r.Matches("Literally anything")).To(gomega.BeTrue()) +} diff --git a/changegroup/rule.go b/changegroup/rule.go new file mode 100644 index 000000000..1b3b75e26 --- /dev/null +++ b/changegroup/rule.go @@ -0,0 +1,191 @@ +package changegroup + +import ( + "time" + + "github.com/flanksource/duty/types" +) + +// GroupingRule declares how the engine should fold matching config_changes +// into a single change_group. Rules are loaded from ScrapeConfig / +// ScrapePlugin specs (field: spec.transform.changes.grouping) and handed +// to the engine via SetRules. +type GroupingRule struct { + Name string `json:"name" yaml:"name"` + Filter string `json:"filter,omitempty" yaml:"filter,omitempty"` + Scope Scope `json:"scope" yaml:"scope"` + Window Duration `json:"window" yaml:"window"` + CloseAfter Duration `json:"closeAfter,omitempty" yaml:"closeAfter,omitempty"` + ChangeTypes []string `json:"changeTypes,omitempty" yaml:"changeTypes,omitempty"` + Key string `json:"key" yaml:"key"` + Details string `json:"details" yaml:"details"` + Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` + Priority int `json:"priority,omitempty" yaml:"priority,omitempty"` + + // Compiled programs are stashed here at Validate() time so per-change + // evaluation skips re-parsing. Not serialized. + filterProgram Program `json:"-" yaml:"-"` + keyProgram Program `json:"-" yaml:"-"` + detailsProgram Program `json:"-" yaml:"-"` + summaryProgram Program `json:"-" yaml:"-"` + + // literalChangeTypes is the expanded set of literal change_type strings + // this rule matches, after pseudo-type expansion. Populated by Validate. + literalChangeTypes map[string]struct{} `json:"-" yaml:"-"` +} + +// Scope narrows which changes the rule can bind together. +type Scope struct { + Kind string `json:"kind" yaml:"kind"` + Field string `json:"field,omitempty" yaml:"field,omitempty"` + Depth int `json:"depth,omitempty" yaml:"depth,omitempty"` +} + +const ( + ScopeSameConfig = "same_config" + ScopeRelated = "related" + ScopeAll = "all" + ScopeByDetailsField = "by_details_field" +) + +// Duration is a time.Duration that marshals to/from a Go duration string +// (e.g. "5s", "30m") so YAML stays human-readable. +type Duration time.Duration + +func (d Duration) Std() time.Duration { return time.Duration(d) } + +func (d Duration) MarshalJSON() ([]byte, error) { + return []byte(`"` + time.Duration(d).String() + `"`), nil +} + +func (d *Duration) UnmarshalJSON(data []byte) error { + s := string(data) + if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' { + s = s[1 : len(s)-1] + } + if s == "" || s == "null" { + *d = 0 + return nil + } + parsed, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(parsed) + return nil +} + +func (d Duration) MarshalYAML() (any, error) { return time.Duration(d).String(), nil } + +func (d *Duration) UnmarshalYAML(unmarshal func(any) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + if s == "" { + *d = 0 + return nil + } + parsed, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(parsed) + return nil +} + +// ExprKind distinguishes the expected return type of a compiled CEL expression. +type ExprKind int + +const ( + ExprBool ExprKind = iota + ExprString + ExprGroupDetails +) + +// Program is an opaque evaluator-specific compiled expression handle. +type Program any + +// Env is the binding set passed to every evaluator call. +type Env struct { + // Change is the triggering change (one-based map of the change row). + Change map[string]any + // Changes is the full list of members currently in the group, including + // the triggering change (appended last). First eval: len == 1. + Changes []map[string]any + // Group is the current persisted ChangeGroup state or nil on first eval. + Group map[string]any + // Flat contains top-level shortcuts mirroring Change.* for single-change rules. + Flat map[string]any +} + +// Evaluator is implemented by an external CEL-backed evaluator (in config-db). +// duty declares only the interface to avoid pulling in CEL as a dependency. +type Evaluator interface { + EvalBool(prog Program, env Env) (bool, error) + EvalString(prog Program, env Env) (string, error) + EvalGroupDetails(prog Program, env Env) (types.GroupType, error) + Compile(expr string, kind ExprKind) (Program, error) +} + +// Validate compiles every expression on the rule and expands pseudo change +// types. It must be called once when the rule is loaded. Returns an error if +// any expression fails to compile or a pseudo type is unknown. +func (r *GroupingRule) Validate(ev Evaluator) error { + if r.Name == "" { + return ErrEmptyRuleName + } + if r.Details == "" { + return ErrMissingDetails + } + if r.Key == "" { + return ErrMissingKey + } + + literals, err := expandChangeTypes(r.ChangeTypes) + if err != nil { + return err + } + r.literalChangeTypes = literals + + if r.Filter != "" { + p, err := ev.Compile(r.Filter, ExprBool) + if err != nil { + return &CompileError{Rule: r.Name, Field: "filter", Err: err} + } + r.filterProgram = p + } + + p, err := ev.Compile(r.Key, ExprString) + if err != nil { + return &CompileError{Rule: r.Name, Field: "key", Err: err} + } + r.keyProgram = p + + p, err = ev.Compile(r.Details, ExprGroupDetails) + if err != nil { + return &CompileError{Rule: r.Name, Field: "details", Err: err} + } + r.detailsProgram = p + + if r.Summary != "" { + p, err := ev.Compile(r.Summary, ExprString) + if err != nil { + return &CompileError{Rule: r.Name, Field: "summary", Err: err} + } + r.summaryProgram = p + } + + return nil +} + +// Matches reports whether the rule's change_types accept the given literal +// change_type string. An empty ChangeTypes list matches everything (after the +// filter expression is also applied). +func (r *GroupingRule) Matches(changeType string) bool { + if len(r.literalChangeTypes) == 0 { + return true + } + _, ok := r.literalChangeTypes[changeType] + return ok +} diff --git a/changegroup/upsert.go b/changegroup/upsert.go new file mode 100644 index 000000000..c72a7f27f --- /dev/null +++ b/changegroup/upsert.go @@ -0,0 +1,197 @@ +package changegroup + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "hash/fnv" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" +) + +// hashKey combines the rule name and raw key expression output into the +// final stored correlation_key. Including the rule name prevents accidental +// collisions between two rules whose key expressions happen to produce the +// same string. +func hashKey(ruleName, rawKey string) string { + h := sha256.Sum256([]byte(ruleName + "|" + rawKey)) + return ruleName + ":" + hex.EncodeToString(h[:]) +} + +// advisoryLockKey converts a (type, correlation_key) pair into an int64 +// suitable for pg_advisory_xact_lock. The lock serializes concurrent upserts +// for the same logical group across workers. +func advisoryLockKey(groupType, correlationKey string) int64 { + h := fnv.New64a() + h.Write([]byte(groupType)) + h.Write([]byte{0}) + h.Write([]byte(correlationKey)) + return int64(h.Sum64()) //nolint:gosec // wraparound is fine for advisory locks +} + +// upsertAndAttach finds-or-creates an open group for the given correlation +// key, evaluates Details and Summary against the full member list (including +// the new change), merges the result into stored details, and sets +// change.GroupID. Runs in a single transaction with an advisory lock. +func (e *Engine) upsertAndAttach( + ctx context.Context, + rule *GroupingRule, + correlationKey string, + change *models.ConfigChange, +) error { + return ctx.DB().Transaction(func(tx *gorm.DB) error { + // Placeholder group type used only for the advisory lock key while we + // look up or create the real row. We don't know the final group type + // until we evaluate Details, so lock on (rule.Name, correlationKey) — + // rule name is already baked into correlationKey, so locking on that + // alone is sufficient. + lockKey := advisoryLockKey(rule.Name, correlationKey) + if err := tx.Exec(`SELECT pg_advisory_xact_lock(?)`, lockKey).Error; err != nil { + return fmt.Errorf("advisory lock: %w", err) + } + + // Find existing open group for this correlation_key, regardless of type. + // The unique index is (type, correlation_key) WHERE status='open', but + // rule name + hash in the key already makes collisions across types + // astronomically unlikely. + var existing models.ChangeGroup + err := tx.Where("correlation_key = ? AND status = ?", correlationKey, models.ChangeGroupStatusOpen). + Take(&existing).Error + + var members []models.ConfigChange + var currentGroup *models.ChangeGroup + if err == nil { + currentGroup = &existing + if err := tx.Where("group_id = ?", existing.ID). + Order("created_at ASC, id ASC"). + Find(&members).Error; err != nil { + return fmt.Errorf("load group members: %w", err) + } + } else if err != gorm.ErrRecordNotFound { + return fmt.Errorf("lookup open group: %w", err) + } + + // Append the triggering change as the last member when computing env. + memberMaps := make([]map[string]any, 0, len(members)+1) + for i := range members { + memberMaps = append(memberMaps, changeAsMap(&members[i])) + } + memberMaps = append(memberMaps, changeAsMap(change)) + + env := Env{ + Change: changeAsMap(change), + Changes: memberMaps, + Flat: changeAsMap(change), + } + if currentGroup != nil { + env.Group = groupAsMap(currentGroup) + } + + // Details is required. + incomingDetails, err := e.evaluator.EvalGroupDetails(rule.detailsProgram, env) + if err != nil { + return &EvalError{Rule: rule.Name, Field: "details", Err: err} + } + if incomingDetails == nil { + return fmt.Errorf("changegroup: rule %q details evaluated to nil", rule.Name) + } + + summary := "" + if rule.summaryProgram != nil { + summary, err = e.evaluator.EvalString(rule.summaryProgram, env) + if err != nil { + return &EvalError{Rule: rule.Name, Field: "summary", Err: err} + } + } + + // Determine the effective group for this attach. + if currentGroup == nil { + createdAt := time.Now().UTC() + if change.CreatedAt != nil { + createdAt = *change.CreatedAt + } + newGroup := models.ChangeGroup{ + ID: uuid.New(), + Type: incomingDetails.Kind(), + Summary: summary, + CorrelationKey: correlationKey, + Source: models.ChangeGroupSourceRule + ":" + rule.Name, + RuleName: &rule.Name, + Status: models.ChangeGroupStatusOpen, + StartedAt: createdAt, + LastMemberAt: createdAt, + MemberCount: 0, // trigger will bump on attach + CreatedAt: createdAt, + UpdatedAt: createdAt, + } + raw, err := json.Marshal(incomingDetails) + if err != nil { + return fmt.Errorf("marshal group details: %w", err) + } + newGroup.Details = types.JSON(raw) + if err := tx.Create(&newGroup).Error; err != nil { + return fmt.Errorf("create change_group: %w", err) + } + currentGroup = &newGroup + } else { + stored, err := currentGroup.TypedDetails() + if err != nil { + return fmt.Errorf("unmarshal stored group details: %w", err) + } + merged, err := Merge(stored, incomingDetails) + if err != nil { + return fmt.Errorf("merge group details: %w", err) + } + raw, err := json.Marshal(merged) + if err != nil { + return fmt.Errorf("marshal merged details: %w", err) + } + updates := map[string]any{ + "details": types.JSON(raw), + "updated_at": time.Now().UTC(), + } + if summary != "" { + updates["summary"] = summary + } + if err := tx.Model(&models.ChangeGroup{}). + Where("id = ?", currentGroup.ID). + Updates(updates).Error; err != nil { + return fmt.Errorf("update change_group: %w", err) + } + } + + // Attach the change to the group. The 047 trigger maintains + // member_count / last_member_at / started_at. + change.GroupID = ¤tGroup.ID + if err := tx.Model(&models.ConfigChange{}). + Where("id = ?", change.ID). + Update("group_id", currentGroup.ID).Error; err != nil { + return fmt.Errorf("attach change to group: %w", err) + } + return nil + }) +} + +// groupAsMap projects a ChangeGroup into the CEL binding shape. +func groupAsMap(g *models.ChangeGroup) map[string]any { + return map[string]any{ + "id": g.ID.String(), + "type": g.Type, + "summary": g.Summary, + "correlation_key": g.CorrelationKey, + "source": g.Source, + "status": g.Status, + "started_at": g.StartedAt, + "ended_at": g.EndedAt, + "last_member_at": g.LastMemberAt, + "member_count": g.MemberCount, + "details": g.Details, + } +} diff --git a/models/change_group.go b/models/change_group.go new file mode 100644 index 000000000..f968895b9 --- /dev/null +++ b/models/change_group.go @@ -0,0 +1,50 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + + "github.com/flanksource/duty/types" +) + +// ChangeGroup represents the change_groups database table — a logical +// grouping of correlated config_changes rows. +type ChangeGroup struct { + ID uuid.UUID `gorm:"primaryKey;column:id;default:generate_ulid()" json:"id"` + Type string `gorm:"column:type" json:"type"` + Summary string `gorm:"column:summary" json:"summary"` + CorrelationKey string `gorm:"column:correlation_key" json:"correlation_key"` + Source string `gorm:"column:source" json:"source"` + RuleName *string `gorm:"column:rule_name" json:"rule_name,omitempty"` + Status string `gorm:"column:status" json:"status"` + StartedAt time.Time `gorm:"column:started_at" json:"started_at"` + EndedAt *time.Time `gorm:"column:ended_at" json:"ended_at,omitempty"` + LastMemberAt time.Time `gorm:"column:last_member_at" json:"last_member_at"` + MemberCount int `gorm:"column:member_count" json:"member_count"` + Details types.JSON `gorm:"column:details" json:"details,omitempty"` + CreatedAt time.Time `gorm:"column:created_at" json:"created_at"` + UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"` +} + +const ( + ChangeGroupStatusOpen = "open" + ChangeGroupStatusClosed = "closed" + + ChangeGroupSourceExplicit = "explicit" + ChangeGroupSourceRule = "rule" +) + +func (ChangeGroup) TableName() string { return "change_groups" } + +func (g ChangeGroup) PK() string { return g.ID.String() } + +// TypedDetails returns the strongly-typed GroupType for the Details column +// by inspecting the "kind" envelope. +func (g ChangeGroup) TypedDetails() (types.GroupType, error) { + if len(g.Details) == 0 { + return nil, nil + } + return types.UnmarshalGroupDetails(json.RawMessage(g.Details)) +} diff --git a/models/changes.go b/models/changes.go index 686d6e61a..73c9dab2e 100644 --- a/models/changes.go +++ b/models/changes.go @@ -31,6 +31,7 @@ type CatalogChange struct { AgentID *uuid.UUID `gorm:"column:agent_id" json:"agent_id"` Path string `gorm:"column:path" json:"path"` InsertedAt *time.Time `gorm:"column:inserted_at" json:"inserted_at,omitempty"` + GroupID *uuid.UUID `gorm:"column:group_id" json:"group_id,omitempty"` } func (c CatalogChange) GetID() string { diff --git a/models/config.go b/models/config.go index 20f37e3d5..81ec0ca33 100644 --- a/models/config.go +++ b/models/config.go @@ -720,6 +720,14 @@ type ConfigChange struct { // Artifacts associated with this change (screenshots, HAR files, etc.) Artifacts []Artifact `gorm:"foreignKey:ConfigChangeID" json:"artifacts,omitempty"` + + // Action is the resolved change mapping action (move-up, copy-up, copy, move, delete, ignore). + // Not stored in the database — populated by the scrape pipeline for diagnostics. + Action string `gorm:"-" json:"action,omitempty"` + + // GroupID is the optional logical grouping this change belongs to. + // Set explicitly by producers or by the grouping engine. + GroupID *uuid.UUID `gorm:"column:group_id" json:"group_id,omitempty"` } func (c ConfigChange) Pretty() api.Text { diff --git a/query/change_groups.go b/query/change_groups.go new file mode 100644 index 000000000..c4db47b9a --- /dev/null +++ b/query/change_groups.go @@ -0,0 +1,150 @@ +package query + +import ( + "strings" + "time" + + "github.com/google/uuid" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" +) + +// ChangeGroupsSearchRequest describes filters for FindChangeGroups. +type ChangeGroupsSearchRequest struct { + // Type is a comma-separated list of change_group.type values. + Type string `query:"type" json:"type,omitempty"` + // Status is open|closed. Empty means any. + Status string `query:"status" json:"status,omitempty"` + // ConfigID restricts to groups that have at least one member belonging to this config_id. + ConfigID *uuid.UUID `query:"config_id" json:"config_id,omitempty"` + // Summary is a case-insensitive substring search on the summary column. + Summary string `query:"summary" json:"summary,omitempty"` + // Since / Until bound started_at. + Since *time.Time `query:"since" json:"since,omitempty"` + Until *time.Time `query:"until" json:"until,omitempty"` + // Pagination. + Page int `query:"page" json:"page,omitempty"` + PageSize int `query:"page_size" json:"page_size,omitempty"` +} + +func (r *ChangeGroupsSearchRequest) setDefaults() { + if r.Page < 1 { + r.Page = 1 + } + if r.PageSize <= 0 || r.PageSize > 500 { + r.PageSize = 50 + } +} + +// FindChangeGroups returns change_groups matching the search request. +func FindChangeGroups(ctx context.Context, req ChangeGroupsSearchRequest) ([]models.ChangeGroup, error) { + req.setDefaults() + + q := ctx.DB().Model(&models.ChangeGroup{}) + + if req.Type != "" { + q = q.Where("type IN ?", strings.Split(req.Type, ",")) + } + if req.Status != "" { + q = q.Where("status = ?", req.Status) + } + if req.Summary != "" { + q = q.Where("summary ILIKE ?", "%"+req.Summary+"%") + } + if req.Since != nil { + q = q.Where("started_at >= ?", *req.Since) + } + if req.Until != nil { + q = q.Where("started_at <= ?", *req.Until) + } + if req.ConfigID != nil { + q = q.Where( + "id IN (SELECT group_id FROM config_changes WHERE config_id = ? AND group_id IS NOT NULL)", + *req.ConfigID, + ) + } + + q = q.Order("started_at DESC"). + Limit(req.PageSize). + Offset((req.Page - 1) * req.PageSize) + + var groups []models.ChangeGroup + if err := q.Find(&groups).Error; err != nil { + return nil, err + } + return groups, nil +} + +// GetChangeGroup loads a single change_group by id. +func GetChangeGroup(ctx context.Context, id uuid.UUID) (*models.ChangeGroup, error) { + var g models.ChangeGroup + if err := ctx.DB().Where("id = ?", id).Take(&g).Error; err != nil { + return nil, err + } + return &g, nil +} + +// GetGroupMembers returns the config_changes rows belonging to the given group. +func GetGroupMembers(ctx context.Context, id uuid.UUID) ([]models.ConfigChange, error) { + var members []models.ConfigChange + if err := ctx.DB(). + Where("group_id = ?", id). + Order("created_at ASC, id ASC"). + Find(&members).Error; err != nil { + return nil, err + } + return members, nil +} + +// ChangeGroupSummary is a row from the change_groups_summary view. +type ChangeGroupSummary struct { + ID uuid.UUID `gorm:"column:id" json:"id"` + Type string `gorm:"column:type" json:"type"` + Summary string `gorm:"column:summary" json:"summary"` + Source string `gorm:"column:source" json:"source"` + RuleName *string `gorm:"column:rule_name" json:"rule_name,omitempty"` + Status string `gorm:"column:status" json:"status"` + StartedAt time.Time `gorm:"column:started_at" json:"started_at"` + EndedAt *time.Time `gorm:"column:ended_at" json:"ended_at,omitempty"` + LastMemberAt time.Time `gorm:"column:last_member_at" json:"last_member_at"` + MemberCount int `gorm:"column:member_count" json:"member_count"` + DistinctConfigCount int `gorm:"column:distinct_config_count" json:"distinct_config_count"` + DurationSeconds float64 `gorm:"column:duration_seconds" json:"duration_seconds"` +} + +func (ChangeGroupSummary) TableName() string { return "change_groups_summary" } + +// FindChangeGroupsSummary returns aggregated rows from change_groups_summary, +// optionally filtered. Filters match FindChangeGroups where applicable. +func FindChangeGroupsSummary(ctx context.Context, req ChangeGroupsSearchRequest) ([]ChangeGroupSummary, error) { + req.setDefaults() + + q := ctx.DB().Table("change_groups_summary") + + if req.Type != "" { + q = q.Where("type IN ?", strings.Split(req.Type, ",")) + } + if req.Status != "" { + q = q.Where("status = ?", req.Status) + } + if req.Summary != "" { + q = q.Where("summary ILIKE ?", "%"+req.Summary+"%") + } + if req.Since != nil { + q = q.Where("started_at >= ?", *req.Since) + } + if req.Until != nil { + q = q.Where("started_at <= ?", *req.Until) + } + + q = q.Order("started_at DESC"). + Limit(req.PageSize). + Offset((req.Page - 1) * req.PageSize) + + var out []ChangeGroupSummary + if err := q.Scan(&out).Error; err != nil { + return nil, err + } + return out, nil +} diff --git a/query/config_changes.go b/query/config_changes.go index c329cc9c5..86d4ca5b5 100644 --- a/query/config_changes.go +++ b/query/config_changes.go @@ -37,6 +37,13 @@ type CatalogChangesSearchRequest struct { Summary string `query:"summary" json:"summary"` Source string `query:"source" json:"source"` + // GroupID restricts results to changes that belong to a specific change_group. + GroupID *uuid.UUID `query:"group_id" json:"group_id,omitempty"` + // GroupType is a comma-separated list of change_group.type values. + GroupType string `query:"group_type" json:"group_type,omitempty"` + // Grouped is a tri-state filter: nil = any, true = only grouped, false = only ungrouped. + Grouped *bool `query:"grouped" json:"grouped,omitempty"` + createdBy *uuid.UUID externalCreatedBy string @@ -295,6 +302,26 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (r } } + if req.GroupID != nil { + clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "group_id"}, Value: *req.GroupID}) + } + + if req.Grouped != nil { + if *req.Grouped { + dbQuery = dbQuery.Where("group_id IS NOT NULL") + } else { + dbQuery = dbQuery.Where("group_id IS NULL") + } + } + + if req.GroupType != "" { + groupTypes := strings.Split(req.GroupType, ",") + dbQuery = dbQuery.Where( + "group_id IN (SELECT id FROM change_groups WHERE type IN ?)", + groupTypes, + ) + } + // Determine table: single UUID uses related_changes_recursive, multi-ID or query uses IN clause table := dbQuery.Table("catalog_changes") if len(configIDs) == 1 { diff --git a/rbac/objects.go b/rbac/objects.go index dc45de7f8..c02cba0ea 100644 --- a/rbac/objects.go +++ b/rbac/objects.go @@ -29,6 +29,8 @@ var dbResourceObjMap = map[string]string{ "canaries": policy.ObjectCanary, "casbin_rule": policy.ObjectAuth, "catalog_changes": policy.ObjectCatalog, + "change_groups": policy.ObjectCatalog, + "change_groups_summary": policy.ObjectCatalog, "change_types": policy.ObjectDatabasePublic, "changes_by_component": policy.ObjectCatalog, "check_component_relationships": policy.ObjectCanary, diff --git a/schema/config.hcl b/schema/config.hcl index eb51e7280..c53762b9f 100644 --- a/schema/config.hcl +++ b/schema/config.hcl @@ -196,6 +196,12 @@ table "config_changes" { default = sql("now()") } + column "group_id" { + null = true + type = uuid + comment = "optional logical grouping of correlated changes; see change_groups table." + } + primary_key { columns = [column.id] } @@ -205,6 +211,12 @@ table "config_changes" { on_update = NO_ACTION on_delete = CASCADE } + foreign_key "config_changes_group_id_fkey" { + columns = [column.group_id] + ref_columns = [table.change_groups.column.id] + on_update = NO_ACTION + on_delete = SET_NULL + } index "config_changes_created_at_brin_idx" { type = BRIN @@ -230,6 +242,112 @@ table "config_changes" { columns = [column.is_pushed] where = "is_pushed IS FALSE" } + index "config_changes_group_id_idx" { + columns = [column.group_id] + where = "group_id IS NOT NULL" + } + index "config_changes_ungrouped_created_at_idx" { + columns = [column.created_at] + where = "group_id IS NULL" + } +} + +table "change_groups" { + schema = schema.public + comment = "logical grouping of correlated config_changes rows (e.g., pod startup burst, same-commit fan-out, temporary grants)." + + column "id" { + null = false + type = uuid + default = sql("generate_ulid()") + } + column "type" { + null = false + type = text + comment = "typed group kind, e.g. Deployment/v1, TemporaryPermission/v1." + } + column "summary" { + null = false + type = text + default = "" + } + column "correlation_key" { + null = false + type = text + comment = "stable key used to find-or-create a group; unique among open groups of the same type." + } + column "source" { + null = false + type = text + default = "rule" + comment = "explicit, or rule:." + } + column "rule_name" { + null = true + type = text + } + column "status" { + null = false + type = text + default = "open" + } + column "started_at" { + null = false + type = timestamptz + default = sql("now()") + } + column "ended_at" { + null = true + type = timestamptz + } + column "last_member_at" { + null = false + type = timestamptz + default = sql("now()") + } + column "member_count" { + null = false + type = int + default = 0 + } + column "details" { + null = true + type = jsonb + } + column "created_at" { + null = false + type = timestamptz + default = sql("now()") + } + column "updated_at" { + null = false + type = timestamptz + default = sql("now()") + } + + primary_key { + columns = [column.id] + } + + index "change_groups_open_type_correlation_key" { + unique = true + columns = [column.type, column.correlation_key] + where = "status = 'open'" + } + index "change_groups_type_started_at_idx" { + columns = [column.type, column.started_at] + } + index "change_groups_last_member_at_idx" { + columns = [column.last_member_at] + } + index "change_groups_created_at_brin_idx" { + type = BRIN + columns = [column.created_at] + } + index "change_groups_details_gin_idx" { + columns = [column.details] + type = GIN + } } table "config_items" { diff --git a/tests/change_groups_test.go b/tests/change_groups_test.go new file mode 100644 index 000000000..8995521c1 --- /dev/null +++ b/tests/change_groups_test.go @@ -0,0 +1,435 @@ +package tests + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + ginkgo "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + + "github.com/flanksource/duty/changegroup" + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/query" + "github.com/flanksource/duty/tests/fixtures/dummy" + "github.com/flanksource/duty/types" +) + +// stubProgram wraps a Go closure so a test can build an Evaluator without CEL. +// The Program value stored on GroupingRule is one of these closures. +type stubProgram struct { + boolFn func(changegroup.Env) (bool, error) + stringFn func(changegroup.Env) (string, error) + detailsFn func(changegroup.Env) (types.GroupType, error) +} + +// stubEvaluator implements changegroup.Evaluator by treating each expression +// string as an opaque key into a map that the test fills out before Validate. +type stubEvaluator struct { + programs map[string]*stubProgram +} + +func newStubEvaluator() *stubEvaluator { return &stubEvaluator{programs: map[string]*stubProgram{}} } + +func (s *stubEvaluator) register(expr string, p *stubProgram) { s.programs[expr] = p } + +func (s *stubEvaluator) Compile(expr string, kind changegroup.ExprKind) (changegroup.Program, error) { + p, ok := s.programs[expr] + if !ok { + return nil, ginkgoError("stub evaluator: no program registered for expression %q", expr) + } + _ = kind + return p, nil +} + +func (s *stubEvaluator) EvalBool(p changegroup.Program, env changegroup.Env) (bool, error) { + return p.(*stubProgram).boolFn(env) +} +func (s *stubEvaluator) EvalString(p changegroup.Program, env changegroup.Env) (string, error) { + return p.(*stubProgram).stringFn(env) +} +func (s *stubEvaluator) EvalGroupDetails(p changegroup.Program, env changegroup.Env) (types.GroupType, error) { + return p.(*stubProgram).detailsFn(env) +} + +func ginkgoError(format string, args ...any) error { + return &stubError{msg: sprintf(format, args...)} +} + +type stubError struct{ msg string } + +func (e *stubError) Error() string { return e.msg } + +func sprintf(format string, args ...any) string { + // Tiny fmt shim so this file doesn't import "fmt" just for one call. + b := make([]byte, 0, len(format)+64) + ai := 0 + for i := 0; i < len(format); i++ { + if format[i] == '%' && i+1 < len(format) && format[i+1] == 'q' && ai < len(args) { + b = append(b, '"') + b = append(b, []byte(args[ai].(string))...) + b = append(b, '"') + i++ + ai++ + continue + } + b = append(b, format[i]) + } + return string(b) +} + +// insertChange persists a config_change row and returns it with ID populated. +func insertChange(ctx context.Context, c models.ConfigChange) models.ConfigChange { + if c.ID == "" { + c.ID = uuid.New().String() + } + if c.CreatedAt == nil { + c.CreatedAt = lo.ToPtr(time.Now().UTC()) + } + Expect(ctx.DB().Create(&c).Error).To(Succeed()) + return c +} + +var _ = ginkgo.Describe("changegroup engine", ginkgo.Ordered, func() { + var ( + ev *stubEvaluator + engine *changegroup.Engine + ) + + ginkgo.BeforeEach(func() { + ev = newStubEvaluator() + engine = nil + }) + + ginkgo.AfterEach(func() { + // Clean rows we inserted so each spec is independent. + Expect(DefaultContext.DB().Exec(`DELETE FROM config_changes WHERE source = 'test-changegroup'`).Error).To(Succeed()) + Expect(DefaultContext.DB().Exec(`DELETE FROM change_groups WHERE source LIKE 'rule:test-%' OR source = 'explicit'`).Error).To(Succeed()) + }) + + ginkgo.It("pod startup: time-bucket groups many changes on one config into one group", func() { + // Rule: same_config + 5s bucket + @changed. + ev.register("startup_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + cfgID := env.Change["config_id"].(string) + return cfgID + ":bucket0", nil + }, + }) + ev.register("startup_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + cfg, _ := uuid.Parse(env.Change["config_id"].(string)) + return types.StartupGroup{ + ConfigID: cfg, + Reason: "test", + RestartCount: len(env.Changes), + }, nil + }, + }) + ev.register("startup_summary", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + return "pod startup burst", nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{ + { + Name: "test-pod-startup", + Scope: changegroup.Scope{Kind: changegroup.ScopeSameConfig}, + Window: changegroup.Duration(5 * time.Second), + ChangeTypes: []string{"UPDATE", "diff"}, + Key: "startup_key", + Details: "startup_details", + Summary: "startup_summary", + }, + }) + Expect(err).ToNot(HaveOccurred()) + + cfg := dummy.NginxIngressPod.ID.String() + for i := 0; i < 5; i++ { + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: cfg, + ChangeType: "diff", + Source: "test-changegroup", + Summary: "startup event", + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + } + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeStartup, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + Expect(groups[0].Summary).To(Equal("pod startup burst")) + Expect(groups[0].MemberCount).To(Equal(5)) + + members, err := query.GetGroupMembers(DefaultContext, groups[0].ID) + Expect(err).ToNot(HaveOccurred()) + Expect(members).To(HaveLen(5)) + }) + + ginkgo.It("deployment fan-out: same image across different configs shares one group", func() { + ev.register("deploy_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + d, _ := env.Change["details"].(types.JSON) + var m map[string]any + if len(d) > 0 { + _ = json.Unmarshal(d, &m) + } + img, _ := m["new_image"].(string) + return img, nil + }, + }) + ev.register("deploy_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + // Aggregate config_ids from all members. + ids := make([]uuid.UUID, 0, len(env.Changes)) + for _, m := range env.Changes { + if id, err := uuid.Parse(m["config_id"].(string)); err == nil { + ids = append(ids, id) + } + } + d, _ := env.Change["details"].(types.JSON) + var details map[string]any + if len(d) > 0 { + _ = json.Unmarshal(d, &details) + } + img, _ := details["new_image"].(string) + return types.DeploymentGroup{ + Image: img, + Version: "v1.2.3", + TargetConfigIDs: ids, + }, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{ + { + Name: "test-deployment-fanout", + Scope: changegroup.Scope{Kind: changegroup.ScopeAll}, + Window: changegroup.Duration(2 * time.Minute), + ChangeTypes: []string{types.ChangeTypeDeployment}, + Key: "deploy_key", + Details: "deploy_details", + }, + }) + Expect(err).ToNot(HaveOccurred()) + + configs := []models.ConfigItem{ + dummy.EC2InstanceA, + dummy.EC2InstanceB, + dummy.NginxIngressPod, + } + for _, ci := range configs { + details := types.JSON(`{"new_image":"registry/app:v1.2.3"}`) + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: ci.ID.String(), + ChangeType: types.ChangeTypeDeployment, + Source: "test-changegroup", + Summary: "deployed " + ci.ID.String(), + Details: details, + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + } + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeDeployment, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + Expect(groups[0].MemberCount).To(Equal(len(configs))) + + typed, err := groups[0].TypedDetails() + Expect(err).ToNot(HaveOccurred()) + dep := typed.(types.DeploymentGroup) + Expect(dep.Image).To(Equal("registry/app:v1.2.3")) + Expect(dep.TargetConfigIDs).To(HaveLen(len(configs))) + }) + + ginkgo.It("temporary permission: grant then revoke close the group with duration", func() { + ev.register("tp_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + d, _ := env.Change["details"].(types.JSON) + var m map[string]any + _ = json.Unmarshal(d, &m) + return m["user_id"].(string) + "|" + m["role_id"].(string), nil + }, + }) + ev.register("tp_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + var grant, revoke *uuid.UUID + for _, m := range env.Changes { + id, _ := uuid.Parse(m["id"].(string)) + switch m["change_type"].(string) { + case types.ChangeTypePermissionAdded: + tmp := id + if grant == nil { + grant = &tmp + } + case types.ChangeTypePermissionRemoved: + tmp := id + if revoke == nil { + revoke = &tmp + } + } + } + d, _ := env.Change["details"].(types.JSON) + var dm map[string]any + _ = json.Unmarshal(d, &dm) + return types.TemporaryPermissionGroup{ + UserID: dm["user_id"].(string), + RoleID: dm["role_id"].(string), + Scope: dm["scope"].(string), + GrantChangeID: grant, + RevokeChangeID: revoke, + }, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{ + { + Name: "test-temporary-permission", + Scope: changegroup.Scope{Kind: changegroup.ScopeByDetailsField, Field: "user_id"}, + Window: changegroup.Duration(720 * time.Hour), + CloseAfter: 0, + ChangeTypes: []string{types.ChangeTypePermissionAdded, types.ChangeTypePermissionRemoved}, + Key: "tp_key", + Details: "tp_details", + }, + }) + Expect(err).ToNot(HaveOccurred()) + + cfg := dummy.EKSCluster.ID.String() + t0 := time.Now().UTC().Add(-time.Hour) + grant := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: cfg, + ChangeType: types.ChangeTypePermissionAdded, + Source: "test-changegroup", + Details: types.JSON(`{"user_id":"alice","role_id":"admin","scope":"cluster-1"}`), + CreatedAt: &t0, + }) + Expect(engine.Evaluate(DefaultContext, &grant)).To(Succeed()) + + t1 := t0.Add(time.Hour) + revoke := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: cfg, + ChangeType: types.ChangeTypePermissionRemoved, + Source: "test-changegroup", + Details: types.JSON(`{"user_id":"alice","role_id":"admin","scope":"cluster-1"}`), + CreatedAt: &t1, + }) + Expect(engine.Evaluate(DefaultContext, &revoke)).To(Succeed()) + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeTemporaryPermission, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + Expect(groups[0].MemberCount).To(Equal(2)) + + typed, err := groups[0].TypedDetails() + Expect(err).ToNot(HaveOccurred()) + tp := typed.(types.TemporaryPermissionGroup) + Expect(tp.GrantChangeID).ToNot(BeNil()) + Expect(tp.RevokeChangeID).ToNot(BeNil()) + Expect(*tp.GrantChangeID).To(Equal(uuid.MustParse(grant.ID))) + Expect(*tp.RevokeChangeID).To(Equal(uuid.MustParse(revoke.ID))) + }) + + ginkgo.It("explicit path: engine leaves producer-assigned group_id alone", func() { + // Rule that would otherwise match — but change already has GroupID. + ev.register("noop_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { return "noop", nil }, + }) + ev.register("noop_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + return types.CustomGroup{Fields: map[string]any{"k": "v"}}, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{{ + Name: "test-should-not-run", + Scope: changegroup.Scope{Kind: changegroup.ScopeAll}, + Window: changegroup.Duration(time.Minute), + Key: "noop_key", + Details: "noop_details", + }}) + Expect(err).ToNot(HaveOccurred()) + + gid, err := changegroup.CreateTyped(DefaultContext, + types.CustomGroup{Fields: map[string]any{"origin": "playbook"}}, + "playbook-driven", + ) + Expect(err).ToNot(HaveOccurred()) + + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: dummy.EC2InstanceA.ID.String(), + ChangeType: "Pulled", + Source: "test-changegroup", + GroupID: &gid, + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + + g, err := query.GetChangeGroup(DefaultContext, gid) + Expect(err).ToNot(HaveOccurred()) + Expect(g.Source).To(Equal(models.ChangeGroupSourceExplicit)) + Expect(g.MemberCount).To(Equal(1), "explicit assign should have been counted by the 047 trigger") + }) + + ginkgo.It("re-evaluation: each attach sees growing changes list", func() { + // Record how many members each evaluation saw. + var observed []int + ev.register("re_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { return "single-key", nil }, + }) + ev.register("re_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + observed = append(observed, len(env.Changes)) + ids := make([]uuid.UUID, 0, len(env.Changes)) + for _, m := range env.Changes { + if id, err := uuid.Parse(m["config_id"].(string)); err == nil { + ids = append(ids, id) + } + } + return types.DeploymentGroup{Image: "x", TargetConfigIDs: ids}, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{{ + Name: "test-re-eval", + Scope: changegroup.Scope{Kind: changegroup.ScopeAll}, + Window: changegroup.Duration(time.Minute), + ChangeTypes: []string{types.ChangeTypeDeployment}, + Key: "re_key", + Details: "re_details", + }}) + Expect(err).ToNot(HaveOccurred()) + + for _, ci := range []models.ConfigItem{dummy.EC2InstanceA, dummy.EC2InstanceB, dummy.NginxIngressPod} { + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: ci.ID.String(), + ChangeType: types.ChangeTypeDeployment, + Source: "test-changegroup", + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + } + + Expect(observed).To(Equal([]int{1, 2, 3})) + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeDeployment, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + typed, err := groups[0].TypedDetails() + Expect(err).ToNot(HaveOccurred()) + Expect(typed.(types.DeploymentGroup).TargetConfigIDs).To(HaveLen(3)) + }) +}) diff --git a/types/config_change_groups.go b/types/config_change_groups.go new file mode 100644 index 000000000..0bad9528c --- /dev/null +++ b/types/config_change_groups.go @@ -0,0 +1,171 @@ +package types + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" +) + +// GroupType is implemented by all typed group detail structs +// for the ChangeGroup.Details field. +type GroupType interface { + Kind() string +} + +// Change group type constants. +const ( + GroupTypeStartup = "Startup/v1" + GroupTypeDeployment = "Deployment/v1" + GroupTypePromotion = "Promotion/v1" + GroupTypeTemporaryPermission = "TemporaryPermission/v1" + GroupTypeIncidentResponse = "IncidentResponse/v1" + GroupTypeCustom = "Custom/v1" +) + +// ConfigChangeGroupDetailsSchema is a union type used for JSON schema generation. +// It is never instantiated at runtime. +type ConfigChangeGroupDetailsSchema struct { + Startup *StartupGroup `json:"Startup/v1,omitempty"` + Deployment *DeploymentGroup `json:"Deployment/v1,omitempty"` + Promotion *PromotionGroup `json:"Promotion/v1,omitempty"` + TemporaryPermission *TemporaryPermissionGroup `json:"TemporaryPermission/v1,omitempty"` + IncidentResponse *IncidentResponseGroup `json:"IncidentResponse/v1,omitempty"` + Custom *CustomGroup `json:"Custom/v1,omitempty"` +} + +type StartupGroup struct { + ConfigID uuid.UUID `json:"config_id"` + Reason string `json:"reason,omitempty"` + RestartCount int `json:"restart_count,omitempty"` +} + +func (g StartupGroup) Kind() string { return GroupTypeStartup } +func (g StartupGroup) MarshalJSON() ([]byte, error) { + type raw StartupGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type DeploymentGroup struct { + Image string `json:"image,omitempty"` + Version string `json:"version,omitempty"` + Commit string `json:"commit,omitempty"` + Strategy string `json:"strategy,omitempty"` + TargetConfigIDs []uuid.UUID `json:"target_config_ids,omitempty" merge:"append"` +} + +func (g DeploymentGroup) Kind() string { return GroupTypeDeployment } +func (g DeploymentGroup) MarshalJSON() ([]byte, error) { + type raw DeploymentGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type PromotionGroup struct { + FromEnvironment string `json:"from_environment,omitempty"` + ToEnvironment string `json:"to_environment,omitempty"` + Version string `json:"version,omitempty"` + Artifact string `json:"artifact,omitempty"` + PromotionChangeID *uuid.UUID `json:"promotion_change_id,omitempty" merge:"firstSet"` + TargetDeploymentIDs []uuid.UUID `json:"target_deployment_ids,omitempty" merge:"append"` +} + +func (g PromotionGroup) Kind() string { return GroupTypePromotion } +func (g PromotionGroup) MarshalJSON() ([]byte, error) { + type raw PromotionGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type TemporaryPermissionGroup struct { + UserID string `json:"user_id,omitempty"` + RoleID string `json:"role_id,omitempty"` + Scope string `json:"scope,omitempty"` + GrantChangeID *uuid.UUID `json:"grant_change_id,omitempty" merge:"firstSet"` + RevokeChangeID *uuid.UUID `json:"revoke_change_id,omitempty" merge:"firstSet"` + DurationSeconds *int64 `json:"duration_seconds,omitempty"` +} + +func (g TemporaryPermissionGroup) Kind() string { return GroupTypeTemporaryPermission } +func (g TemporaryPermissionGroup) MarshalJSON() ([]byte, error) { + type raw TemporaryPermissionGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type IncidentResponseGroup struct { + IncidentID string `json:"incident_id,omitempty"` + OpenedAt time.Time `json:"opened_at,omitempty" merge:"min"` + ClosedAt time.Time `json:"closed_at,omitempty" merge:"max"` + PlaybookRunIDs []uuid.UUID `json:"playbook_run_ids,omitempty" merge:"append"` +} + +func (g IncidentResponseGroup) Kind() string { return GroupTypeIncidentResponse } +func (g IncidentResponseGroup) MarshalJSON() ([]byte, error) { + type raw IncidentResponseGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type CustomGroup struct { + Fields map[string]any `json:"fields,omitempty" merge:"mapMerge"` +} + +func (g CustomGroup) Kind() string { return GroupTypeCustom } +func (g CustomGroup) MarshalJSON() ([]byte, error) { + type raw CustomGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +// UnmarshalGroupDetails inspects the "kind" envelope and returns the matching +// concrete GroupType value. +func UnmarshalGroupDetails(raw json.RawMessage) (GroupType, error) { + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + + var envelope struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return nil, fmt.Errorf("decode group details envelope: %w", err) + } + + switch envelope.Kind { + case GroupTypeStartup: + var g StartupGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeDeployment: + var g DeploymentGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypePromotion: + var g PromotionGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeTemporaryPermission: + var g TemporaryPermissionGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeIncidentResponse: + var g IncidentResponseGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeCustom: + var g CustomGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + default: + return nil, fmt.Errorf("unknown group kind %q", envelope.Kind) + } +} diff --git a/types/config_change_groups_test.go b/types/config_change_groups_test.go new file mode 100644 index 000000000..c0cdca5dd --- /dev/null +++ b/types/config_change_groups_test.go @@ -0,0 +1,117 @@ +package types + +import ( + "encoding/json" + "testing" + "time" + + "github.com/google/uuid" + "github.com/onsi/gomega" +) + +func TestGroupDetailsRoundTrip(t *testing.T) { + cfg := uuid.New() + grant := uuid.New() + revoke := uuid.New() + dur := int64(3600) + now := time.Now().UTC().Truncate(time.Second) + + cases := []struct { + name string + in GroupType + }{ + { + name: "startup", + in: StartupGroup{ConfigID: cfg, Reason: "CrashLoopBackOff", RestartCount: 3}, + }, + { + name: "deployment", + in: DeploymentGroup{ + Image: "registry/app:v1.2.3", + Version: "v1.2.3", + Commit: "abc123", + Strategy: "RollingUpdate", + TargetConfigIDs: []uuid.UUID{uuid.New(), uuid.New()}, + }, + }, + { + name: "promotion", + in: PromotionGroup{ + FromEnvironment: "dev", + ToEnvironment: "prod", + Version: "v1.2.3", + Artifact: "app", + PromotionChangeID: &grant, + TargetDeploymentIDs: []uuid.UUID{uuid.New()}, + }, + }, + { + name: "temporary_permission", + in: TemporaryPermissionGroup{ + UserID: "user-1", + RoleID: "admin", + Scope: "cluster-1", + GrantChangeID: &grant, + RevokeChangeID: &revoke, + DurationSeconds: &dur, + }, + }, + { + name: "incident_response", + in: IncidentResponseGroup{ + IncidentID: "INC-42", + OpenedAt: now, + ClosedAt: now.Add(time.Hour), + PlaybookRunIDs: []uuid.UUID{uuid.New()}, + }, + }, + { + name: "custom", + in: CustomGroup{Fields: map[string]any{"foo": "bar", "n": float64(42)}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + raw, err := json.Marshal(tc.in) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // envelope must carry the kind. + var envelope struct { + Kind string `json:"kind"` + } + g.Expect(json.Unmarshal(raw, &envelope)).To(gomega.Succeed()) + g.Expect(envelope.Kind).To(gomega.Equal(tc.in.Kind())) + + got, err := UnmarshalGroupDetails(raw) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got.Kind()).To(gomega.Equal(tc.in.Kind())) + + // Round-tripped value re-marshals to the same JSON. + reraw, err := json.Marshal(got) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(reraw).To(gomega.MatchJSON(raw)) + }) + } +} + +func TestUnmarshalGroupDetailsEmpty(t *testing.T) { + g := gomega.NewWithT(t) + + got, err := UnmarshalGroupDetails(nil) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.BeNil()) + + got, err = UnmarshalGroupDetails(json.RawMessage("null")) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.BeNil()) +} + +func TestUnmarshalGroupDetailsUnknownKind(t *testing.T) { + g := gomega.NewWithT(t) + + _, err := UnmarshalGroupDetails(json.RawMessage(`{"kind":"Nope/v1"}`)) + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("unknown group kind"))) +} diff --git a/views/030_config_changes.sql b/views/030_config_changes.sql index c2a217e7a..347f886cd 100644 --- a/views/030_config_changes.sql +++ b/views/030_config_changes.sql @@ -27,7 +27,8 @@ BEGIN patches = NEW.patches, severity = NEW.severity, source = NEW.source, - summary = NEW.summary + summary = NEW.summary, + group_id = NEW.group_id WHERE id = NEW.id; diff --git a/views/047_change_groups.sql b/views/047_change_groups.sql new file mode 100644 index 000000000..dabec2dd2 --- /dev/null +++ b/views/047_change_groups.sql @@ -0,0 +1,69 @@ +-- dependsOn: functions/drop.sql, views/030_config_changes.sql + +-- Maintains change_groups.member_count, last_member_at, ended_at and started_at +-- when a config_changes row is attached to a group via group_id. +-- Runs on INSERT and on UPDATE OF group_id only, so the dedup UPDATE path in +-- config_changes_update_trigger() does not re-evaluate membership. +CREATE OR REPLACE FUNCTION change_groups_maintain_members() +RETURNS TRIGGER AS $$ +BEGIN + IF NEW.group_id IS NULL THEN + RETURN NEW; + END IF; + + IF TG_OP = 'UPDATE' AND OLD.group_id IS NOT DISTINCT FROM NEW.group_id THEN + RETURN NEW; + END IF; + + UPDATE change_groups g + SET + member_count = g.member_count + 1, + last_member_at = GREATEST(g.last_member_at, NEW.created_at), + ended_at = CASE + WHEN g.ended_at IS NULL THEN NULL + ELSE GREATEST(g.ended_at, NEW.created_at) + END, + started_at = CASE + WHEN g.member_count = 0 THEN NEW.created_at + ELSE LEAST(g.started_at, NEW.created_at) + END, + updated_at = now() + WHERE g.id = NEW.group_id; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_change_groups_maintain_members_ins ON config_changes; +CREATE TRIGGER trg_change_groups_maintain_members_ins +AFTER INSERT ON config_changes +FOR EACH ROW +EXECUTE FUNCTION change_groups_maintain_members(); + +DROP TRIGGER IF EXISTS trg_change_groups_maintain_members_upd ON config_changes; +CREATE TRIGGER trg_change_groups_maintain_members_upd +AFTER UPDATE OF group_id ON config_changes +FOR EACH ROW +EXECUTE FUNCTION change_groups_maintain_members(); + +-- Aggregated view used by the query layer / UI. +CREATE OR REPLACE VIEW change_groups_summary AS +SELECT + g.id, + g.type, + g.summary, + g.source, + g.rule_name, + g.status, + g.started_at, + g.ended_at, + g.last_member_at, + g.member_count, + COUNT(DISTINCT cc.config_id) AS distinct_config_count, + EXTRACT(EPOCH FROM (COALESCE(g.ended_at, g.last_member_at) - g.started_at)) AS duration_seconds, + g.details, + g.created_at, + g.updated_at +FROM change_groups g +LEFT JOIN config_changes cc ON cc.group_id = g.id +GROUP BY g.id;