diff --git a/.gitignore b/.gitignore index f5bf7b1a9..70b99bdd7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ out.html specs/*.md generate-openapi .todos/ +ginkgo*.json diff --git a/changegroup/closer.go b/changegroup/closer.go new file mode 100644 index 000000000..e3564628c --- /dev/null +++ b/changegroup/closer.go @@ -0,0 +1,108 @@ +package changegroup + +import ( + "encoding/json" + "time" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" +) + +// CloseStaleGroups closes open change_groups whose last_member_at is older +// than the effective close window for the group's rule. Called periodically +// by StartCloser. +// +// For TemporaryPermissionGroup specifically, this also computes +// DurationSeconds from started_at/ended_at at close time. +func (e *Engine) CloseStaleGroups(ctx context.Context, now time.Time) (int, error) { + // Build the rule → CloseAfter lookup snapshot. + e.mu.RLock() + closeAfterByRule := make(map[string]time.Duration, len(e.rules)) + for _, r := range e.rules { + ca := r.CloseAfter.Std() + if ca == 0 { + // 0 means "never time out" — skip timeout-close for this rule. + continue + } + closeAfterByRule[r.Name] = ca + } + e.mu.RUnlock() + + if len(closeAfterByRule) == 0 { + return 0, nil + } + + var candidates []models.ChangeGroup + if err := ctx.DB(). + Where("status = ?", models.ChangeGroupStatusOpen). + Find(&candidates).Error; err != nil { + return 0, err + } + + closed := 0 + for i := range candidates { + g := &candidates[i] + if g.RuleName == nil { + continue + } + window, ok := closeAfterByRule[*g.RuleName] + if !ok { + continue + } + if now.Sub(g.LastMemberAt) < window { + continue + } + if err := finalizeClose(ctx, g, g.LastMemberAt); err != nil { + return closed, err + } + closed++ + } + return closed, nil +} + +// finalizeClose writes the terminal state for a group: status=closed, +// ended_at set, and — for TemporaryPermissionGroup — DurationSeconds computed. +func finalizeClose(ctx context.Context, g *models.ChangeGroup, endedAt time.Time) error { + updates := map[string]any{ + "status": models.ChangeGroupStatusClosed, + "ended_at": endedAt, + "updated_at": time.Now().UTC(), + } + + stored, err := g.TypedDetails() + if err == nil && stored != nil { + if tp, ok := stored.(types.TemporaryPermissionGroup); ok { + dur := int64(endedAt.Sub(g.StartedAt).Seconds()) + tp.DurationSeconds = &dur + raw, err := json.Marshal(tp) + if err == nil { + updates["details"] = types.JSON(raw) + } + } + } + + return ctx.DB().Model(&models.ChangeGroup{}). + Where("id = ?", g.ID). + Updates(updates).Error +} + +// StartCloser runs CloseStaleGroups on a ticker until ctx is cancelled. +// Intended to be spawned once per process by the job scheduler. +func (e *Engine) StartCloser(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = 30 * time.Second + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-ticker.C: + _, _ = e.CloseStaleGroups(ctx, now) + } + } + }() +} diff --git a/changegroup/engine.go b/changegroup/engine.go new file mode 100644 index 000000000..5d970ac26 --- /dev/null +++ b/changegroup/engine.go @@ -0,0 +1,175 @@ +package changegroup + +import ( + "sort" + "sync" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" +) + +// Engine evaluates GroupingRules against incoming config_changes and keeps +// change_groups in sync. It is safe for concurrent use once rules are loaded. +type Engine struct { + mu sync.RWMutex + rules []*GroupingRule + evaluator Evaluator +} + +// New returns an Engine pre-loaded with the given rules and evaluator. +// The caller is responsible for calling Validate on each rule before passing +// it in — New re-validates and returns an error if any rule fails. +func New(evaluator Evaluator, rules []GroupingRule) (*Engine, error) { + e := &Engine{evaluator: evaluator} + if err := e.SetRules(rules); err != nil { + return nil, err + } + return e, nil +} + +// SetRules replaces the engine's rule set atomically. All rules are +// (re-)validated; if any rule fails, the previous rule set is preserved and +// the error is returned. +func (e *Engine) SetRules(rules []GroupingRule) error { + if e.evaluator == nil { + // Allow rule loading with no evaluator only if every rule has empty + // CEL expressions (currently never true). Require evaluator. + return ErrMissingEvaluator + } + + compiled := make([]*GroupingRule, 0, len(rules)) + for i := range rules { + r := rules[i] + if err := r.Validate(e.evaluator); err != nil { + return err + } + compiled = append(compiled, &r) + } + sort.SliceStable(compiled, func(i, j int) bool { + return compiled[i].Priority > compiled[j].Priority + }) + + e.mu.Lock() + e.rules = compiled + e.mu.Unlock() + return nil +} + +// Rules returns a snapshot of the currently loaded rules for inspection/tests. +func (e *Engine) Rules() []*GroupingRule { + e.mu.RLock() + defer e.mu.RUnlock() + out := make([]*GroupingRule, len(e.rules)) + copy(out, e.rules) + return out +} + +// Evaluate runs the rule engine against a single already-persisted +// config_changes row. Matching rules will create or update a change_group and +// set change.GroupID. If the change already has a GroupID, Evaluate is a no-op +// (producers are trusted). +func (e *Engine) Evaluate(ctx context.Context, change *models.ConfigChange) error { + if change == nil { + return nil + } + if change.GroupID != nil { + return nil // explicit path — respect producer assignment + } + + e.mu.RLock() + rules := e.rules + e.mu.RUnlock() + + for _, rule := range rules { + matched, err := e.tryRule(ctx, rule, change) + if err != nil { + return err + } + if matched { + return nil + } + } + return nil +} + +// tryRule attempts to apply a single rule to the change. Returns (true, nil) +// on a successful attach, (false, nil) on a non-match. +func (e *Engine) tryRule(ctx context.Context, rule *GroupingRule, change *models.ConfigChange) (bool, error) { + if !rule.Matches(change.ChangeType) { + return false, nil + } + + env := buildSingleChangeEnv(change) + + if rule.filterProgram != nil { + ok, err := e.evaluator.EvalBool(rule.filterProgram, env) + if err != nil { + return false, &EvalError{Rule: rule.Name, Field: "filter", Err: err} + } + if !ok { + return false, nil + } + } + + rawKey, err := e.evaluator.EvalString(rule.keyProgram, env) + if err != nil { + return false, &EvalError{Rule: rule.Name, Field: "key", Err: err} + } + if rawKey == "" { + return false, nil + } + correlationKey := hashKey(rule.Name, rawKey) + + if err := e.upsertAndAttach(ctx, rule, correlationKey, change); err != nil { + return false, err + } + return true, nil +} + +// buildSingleChangeEnv creates an Env whose Changes contains only the +// triggering change (plus the flat shortcuts). Used on the first call; the +// upsert path rebuilds the env with all persisted members before re-running +// Details / Summary. +func buildSingleChangeEnv(change *models.ConfigChange) Env { + m := changeAsMap(change) + return Env{ + Change: m, + Changes: []map[string]any{m}, + Flat: m, + } +} + +// changeAsMap projects a ConfigChange into the CEL binding shape. +func changeAsMap(c *models.ConfigChange) map[string]any { + var groupID any + if c.GroupID != nil { + groupID = c.GroupID.String() + } + return map[string]any{ + "id": c.ID, + "external_id": c.ExternalID, + "external_change_id": derefString(c.ExternalChangeID), + "config_id": c.ConfigID, + "change_type": c.ChangeType, + "severity": string(c.Severity), + "source": c.Source, + "summary": c.Summary, + "patches": c.Patches, + "diff": c.Diff, + "fingerprint": c.Fingerprint, + "details": c.Details, + "created_at": c.CreatedAt, + "created_by": c.CreatedBy, + "external_created_by": derefString(c.ExternalCreatedBy), + "count": c.Count, + "group_id": groupID, + } +} + +func derefString(s *string) string { + if s == nil { + return "" + } + return *s +} + diff --git a/changegroup/errors.go b/changegroup/errors.go new file mode 100644 index 000000000..b2ad170a9 --- /dev/null +++ b/changegroup/errors.go @@ -0,0 +1,41 @@ +package changegroup + +import ( + "errors" + "fmt" +) + +var ( + ErrEmptyRuleName = errors.New("changegroup: rule name is required") + ErrMissingDetails = errors.New("changegroup: rule details expression is required") + ErrMissingKey = errors.New("changegroup: rule key expression is required") + ErrUnknownPseudo = errors.New("changegroup: unknown pseudo change type") + ErrMissingEvaluator = errors.New("changegroup: evaluator must be set before rules are loaded") +) + +// EvalError wraps an evaluator runtime failure with the originating rule and field. +type EvalError struct { + Rule string + Field string + Err error +} + +func (e *EvalError) Error() string { + return fmt.Sprintf("changegroup: eval rule %q field %q: %v", e.Rule, e.Field, e.Err) +} + +func (e *EvalError) Unwrap() error { return e.Err } + +// CompileError wraps an evaluator compile failure with the originating rule +// and field so operators can pinpoint their YAML. +type CompileError struct { + Rule string + Field string + Err error +} + +func (e *CompileError) Error() string { + return fmt.Sprintf("changegroup: compile rule %q field %q: %v", e.Rule, e.Field, e.Err) +} + +func (e *CompileError) Unwrap() error { return e.Err } diff --git a/changegroup/explicit.go b/changegroup/explicit.go new file mode 100644 index 000000000..dff1f52fa --- /dev/null +++ b/changegroup/explicit.go @@ -0,0 +1,80 @@ +package changegroup + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" +) + +// Create inserts a new change_group with an explicit source. Producers call +// this when they know upfront which changes belong together (e.g. a playbook +// emits a coordinated deployment across several configs). +// +// The correlation key defaults to the group id if empty — explicit groups are +// keyed on identity, not content. +func Create(ctx context.Context, group models.ChangeGroup) (uuid.UUID, error) { + if group.ID == uuid.Nil { + group.ID = uuid.New() + } + if group.CorrelationKey == "" { + group.CorrelationKey = "explicit:" + group.ID.String() + } + if group.Source == "" { + group.Source = models.ChangeGroupSourceExplicit + } + if group.Status == "" { + group.Status = models.ChangeGroupStatusOpen + } + now := time.Now().UTC() + if group.StartedAt.IsZero() { + group.StartedAt = now + } + if group.LastMemberAt.IsZero() { + group.LastMemberAt = group.StartedAt + } + if group.CreatedAt.IsZero() { + group.CreatedAt = now + } + if group.UpdatedAt.IsZero() { + group.UpdatedAt = now + } + + if err := ctx.DB().Create(&group).Error; err != nil { + return uuid.Nil, err + } + return group.ID, nil +} + +// CreateTyped is a convenience that builds a ChangeGroup from a typed +// GroupType details value plus the usual metadata fields. +func CreateTyped( + ctx context.Context, + kind types.GroupType, + summary string, +) (uuid.UUID, error) { + raw, err := json.Marshal(kind) + if err != nil { + return uuid.Nil, err + } + return Create(ctx, models.ChangeGroup{ + Type: kind.Kind(), + Summary: summary, + Details: types.JSON(raw), + }) +} + +// Assign attaches the given already-persisted config_changes rows to the +// given group. The 047 trigger maintains member_count and time bounds. +func Assign(ctx context.Context, groupID uuid.UUID, changeIDs ...string) error { + if len(changeIDs) == 0 { + return nil + } + return ctx.DB().Model(&models.ConfigChange{}). + Where("id IN ?", changeIDs). + Update("group_id", groupID).Error +} diff --git a/changegroup/merge.go b/changegroup/merge.go new file mode 100644 index 000000000..7503be050 --- /dev/null +++ b/changegroup/merge.go @@ -0,0 +1,183 @@ +package changegroup + +import ( + "fmt" + "reflect" + "time" + + "github.com/flanksource/duty/types" +) + +// Merge combines new typed group details into the existing stored value, +// using `merge:"..."` struct tags on the concrete GroupType to decide per-field +// strategy. Both arguments must be non-nil and of the same concrete type. +// +// Strategies: +// - append: dedupe-append slices. +// - firstSet: keep the existing value if it was already set; otherwise take new. +// - min / max: reduce on comparable scalars (time.Time, numbers, strings). +// - (default): last-write-wins for scalars; new slice/map replaces existing +// iff the new value is non-empty, otherwise the old value is preserved. +// +// Nil pointers / zero slices on the "new" side never clobber a populated +// existing value — this lets CEL omit fields a given member doesn't know about. +func Merge(existing, incoming types.GroupType) (types.GroupType, error) { + if existing == nil { + return incoming, nil + } + if incoming == nil { + return existing, nil + } + if existing.Kind() != incoming.Kind() { + return nil, fmt.Errorf("changegroup: cannot merge %q with %q", existing.Kind(), incoming.Kind()) + } + + // Work on addressable copies so reflect.Set works even when callers pass + // value types (e.g. types.DeploymentGroup{}). + ex := reflect.New(reflect.TypeOf(existing)).Elem() + ex.Set(reflect.ValueOf(existing)) + in := reflect.ValueOf(incoming) + + if ex.Kind() != reflect.Struct { + return nil, fmt.Errorf("changegroup: cannot merge non-struct GroupType %q", existing.Kind()) + } + + for i := 0; i < ex.NumField(); i++ { + field := ex.Type().Field(i) + if !field.IsExported() { + continue + } + strategy := field.Tag.Get("merge") + exField := ex.Field(i) + inField := in.Field(i) + mergeField(exField, inField, strategy) + } + + return ex.Interface().(types.GroupType), nil +} + +func mergeField(dst, src reflect.Value, strategy string) { + // Never clobber with an explicit zero value on the incoming side — this + // lets rules omit fields they don't care about. + if isZero(src) { + return + } + + switch strategy { + case "append": + mergeAppend(dst, src) + case "firstSet": + if isZero(dst) { + dst.Set(src) + } + case "min": + if isZero(dst) || lessThan(src, dst) { + dst.Set(src) + } + case "max": + if isZero(dst) || lessThan(dst, src) { + dst.Set(src) + } + case "mapMerge": + mergeMap(dst, src) + default: + // last-write-wins for scalars; replace for slices/maps when src is non-empty. + dst.Set(src) + } +} + +// mergeMap performs a per-key merge of two maps of the same type. Keys only +// in dst are preserved; keys in both take the src value (last-write-wins). +func mergeMap(dst, src reflect.Value) { + if dst.Kind() != reflect.Map || src.Kind() != reflect.Map { + dst.Set(src) + return + } + if dst.IsNil() { + dst.Set(reflect.MakeMapWithSize(dst.Type(), src.Len())) + } + iter := src.MapRange() + for iter.Next() { + dst.SetMapIndex(iter.Key(), iter.Value()) + } +} + +// mergeAppend performs a dedupe-append of src slice elements into dst. +// Both must be slices of the same element kind. +func mergeAppend(dst, src reflect.Value) { + if dst.Kind() != reflect.Slice || src.Kind() != reflect.Slice { + // Mis-tagged field; fall back to replace. + dst.Set(src) + return + } + seen := make(map[any]struct{}, dst.Len()+src.Len()) + result := reflect.MakeSlice(dst.Type(), 0, dst.Len()+src.Len()) + for i := 0; i < dst.Len(); i++ { + v := dst.Index(i).Interface() + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + result = reflect.Append(result, dst.Index(i)) + } + for i := 0; i < src.Len(); i++ { + v := src.Index(i).Interface() + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + result = reflect.Append(result, src.Index(i)) + } + dst.Set(result) +} + +// isZero reports whether v holds its type's zero value. For pointers, a nil +// pointer is zero; for slices/maps, len == 0 is zero. +func isZero(v reflect.Value) bool { + switch v.Kind() { + case reflect.Ptr, reflect.Interface: + return v.IsNil() + case reflect.Slice, reflect.Map: + return v.Len() == 0 + default: + return v.IsZero() + } +} + +// lessThan compares two values of the same comparable type. It supports +// time.Time, signed/unsigned integers, floats and strings. For any other +// type it returns false (conservative: don't overwrite). +func lessThan(a, b reflect.Value) bool { + // Dereference pointers. + for a.Kind() == reflect.Ptr { + if a.IsNil() { + return true + } + a = a.Elem() + } + for b.Kind() == reflect.Ptr { + if b.IsNil() { + return false + } + b = b.Elem() + } + + if t, ok := a.Interface().(time.Time); ok { + if u, ok := b.Interface().(time.Time); ok { + return t.Before(u) + } + } + + switch a.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return a.Int() < b.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return a.Uint() < b.Uint() + case reflect.Float32, reflect.Float64: + return a.Float() < b.Float() + case reflect.String: + return a.String() < b.String() + } + return false +} + diff --git a/changegroup/merge_test.go b/changegroup/merge_test.go new file mode 100644 index 000000000..2c382797a --- /dev/null +++ b/changegroup/merge_test.go @@ -0,0 +1,143 @@ +package changegroup + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/onsi/gomega" + + "github.com/flanksource/duty/types" +) + +func TestMergeAppend(t *testing.T) { + g := gomega.NewWithT(t) + + a := uuid.New() + b := uuid.New() + c := uuid.New() + + existing := types.DeploymentGroup{ + Image: "registry/app:v1", + TargetConfigIDs: []uuid.UUID{a, b}, + } + incoming := types.DeploymentGroup{ + Image: "registry/app:v1", + TargetConfigIDs: []uuid.UUID{b, c}, // b is a duplicate + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + dep := out.(types.DeploymentGroup) + g.Expect(dep.TargetConfigIDs).To(gomega.Equal([]uuid.UUID{a, b, c})) + g.Expect(dep.Image).To(gomega.Equal("registry/app:v1")) +} + +func TestMergeFirstSet(t *testing.T) { + g := gomega.NewWithT(t) + + grant := uuid.New() + revoke := uuid.New() + otherGrant := uuid.New() + + existing := types.TemporaryPermissionGroup{ + UserID: "u1", + GrantChangeID: &grant, + } + // "incoming" arrives with a new revoke and a (wrong) new grant id — firstSet + // must keep the old grant id. + incoming := types.TemporaryPermissionGroup{ + UserID: "u1", + GrantChangeID: &otherGrant, + RevokeChangeID: &revoke, + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + tp := out.(types.TemporaryPermissionGroup) + g.Expect(tp.GrantChangeID).To(gomega.Equal(&grant)) + g.Expect(tp.RevokeChangeID).To(gomega.Equal(&revoke)) +} + +func TestMergeMinMax(t *testing.T) { + g := gomega.NewWithT(t) + + t1 := time.Date(2026, 1, 1, 10, 0, 0, 0, time.UTC) + t2 := time.Date(2026, 1, 1, 11, 0, 0, 0, time.UTC) + t3 := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + + existing := types.IncidentResponseGroup{ + IncidentID: "INC-1", + OpenedAt: t2, + ClosedAt: t2, + PlaybookRunIDs: []uuid.UUID{uuid.New()}, + } + incoming := types.IncidentResponseGroup{ + IncidentID: "INC-1", + OpenedAt: t1, // earlier → wins min + ClosedAt: t3, // later → wins max + PlaybookRunIDs: []uuid.UUID{uuid.New()}, + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + ir := out.(types.IncidentResponseGroup) + g.Expect(ir.OpenedAt).To(gomega.Equal(t1)) + g.Expect(ir.ClosedAt).To(gomega.Equal(t3)) + g.Expect(ir.PlaybookRunIDs).To(gomega.HaveLen(2)) +} + +func TestMergeIgnoresZeroIncoming(t *testing.T) { + g := gomega.NewWithT(t) + + existing := types.DeploymentGroup{ + Image: "registry/app:v1", + Version: "v1", + } + // Incoming omits Version — existing Version must survive. + incoming := types.DeploymentGroup{ + Image: "registry/app:v2", + } + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + dep := out.(types.DeploymentGroup) + g.Expect(dep.Image).To(gomega.Equal("registry/app:v2"), "scalar last-write-wins") + g.Expect(dep.Version).To(gomega.Equal("v1"), "zero incoming must not clobber") +} + +func TestMergeKindMismatch(t *testing.T) { + g := gomega.NewWithT(t) + + _, err := Merge( + types.DeploymentGroup{Image: "a"}, + types.PromotionGroup{Version: "v1"}, + ) + g.Expect(err).To(gomega.HaveOccurred()) +} + +func TestMergeCustomGroup(t *testing.T) { + g := gomega.NewWithT(t) + + existing := types.CustomGroup{Fields: map[string]any{"a": 1, "b": 2}} + incoming := types.CustomGroup{Fields: map[string]any{"b": 20, "c": 3}} + + out, err := Merge(existing, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + cg := out.(types.CustomGroup) + g.Expect(cg.Fields).To(gomega.Equal(map[string]any{"a": 1, "b": 20, "c": 3})) +} + +func TestMergeNilFirstMember(t *testing.T) { + g := gomega.NewWithT(t) + + incoming := types.DeploymentGroup{Image: "a"} + out, err := Merge(nil, incoming) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(out).To(gomega.Equal(incoming)) +} diff --git a/changegroup/pseudo.go b/changegroup/pseudo.go new file mode 100644 index 000000000..f910c1e94 --- /dev/null +++ b/changegroup/pseudo.go @@ -0,0 +1,118 @@ +package changegroup + +import ( + "fmt" + "sort" + + "github.com/flanksource/duty/types" +) + +// Pseudo change-type identifiers used in rule DSL. Entries in +// GroupingRule.ChangeTypes starting with "@" are expanded via ExpandPseudo. +const ( + PseudoCreated = "@created" + PseudoChanged = "@changed" + PseudoDeleted = "@deleted" + PseudoHealthy = "@healthy" + PseudoUnhealthy = "@unhealthy" + PseudoPlaybook = "@playbook" +) + +// pseudoMap defines the literal change_type set for each pseudo identifier. +// Keyed on the pseudo string (leading "@" preserved). Kept private to this +// file; callers use ExpandPseudo. +var pseudoMap = map[string]map[string]struct{}{ + PseudoCreated: setOf( + types.ChangeTypeCreate, + "Created", + types.ChangeTypeUserCreated, + types.ChangeTypeRegisterNode, + types.ChangeTypeRunInstances, + ), + PseudoDeleted: setOf( + types.ChangeTypeDelete, + "Deleted", + types.ChangeTypeUserDeleted, + ), + PseudoHealthy: setOf( + "Healthy", + types.ChangeTypeBackupCompleted, + types.ChangeTypeBackupRestored, + types.ChangeTypePipelineRunCompleted, + types.ChangeTypePlaybookCompleted, + types.ChangeTypeCertificateRenewed, + ), + PseudoUnhealthy: setOf( + "Unhealthy", + types.ChangeTypeBackupFailed, + types.ChangeTypePipelineRunFailed, + types.ChangeTypePlaybookFailed, + types.ChangeTypeCertificateExpired, + ), + PseudoChanged: setOf( + types.ChangeTypeUpdate, + types.ChangeTypeDiff, + types.ChangeTypeDeployment, + types.ChangeTypePromotion, + types.ChangeTypeRollback, + types.ChangeTypeScaling, + types.ChangeTypeCostChange, + types.ChangeTypePermissionAdded, + types.ChangeTypePermissionRemoved, + types.ChangeTypeGroupMemberAdded, + types.ChangeTypeGroupMemberRemoved, + ), + PseudoPlaybook: setOf( + types.ChangeTypePlaybookStarted, + types.ChangeTypePlaybookCompleted, + types.ChangeTypePlaybookFailed, + ), +} + +func setOf(items ...string) map[string]struct{} { + m := make(map[string]struct{}, len(items)) + for _, it := range items { + m[it] = struct{}{} + } + return m +} + +// ExpandPseudo returns the sorted list of literal change_type strings that +// the given pseudo identifier covers. Returns an error for unknown pseudo +// identifiers. +func ExpandPseudo(pseudo string) ([]string, error) { + set, ok := pseudoMap[pseudo] + if !ok { + return nil, fmt.Errorf("%w: %q", ErrUnknownPseudo, pseudo) + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// expandChangeTypes builds the deduped literal change_type set for a rule's +// ChangeTypes slice. Entries starting with "@" expand via ExpandPseudo; +// everything else is taken as a literal. +func expandChangeTypes(entries []string) (map[string]struct{}, error) { + out := make(map[string]struct{}, len(entries)) + for _, e := range entries { + if e == "" { + continue + } + if e[0] == '@' { + lits, err := ExpandPseudo(e) + if err != nil { + return nil, err + } + for _, l := range lits { + out[l] = struct{}{} + } + continue + } + out[e] = struct{}{} + } + return out, nil +} diff --git a/changegroup/pseudo_test.go b/changegroup/pseudo_test.go new file mode 100644 index 000000000..a0275bac6 --- /dev/null +++ b/changegroup/pseudo_test.go @@ -0,0 +1,91 @@ +package changegroup + +import ( + "errors" + "testing" + + "github.com/onsi/gomega" + + "github.com/flanksource/duty/types" +) + +func TestExpandPseudo(t *testing.T) { + cases := []struct { + pseudo string + want []string + }{ + { + pseudo: PseudoCreated, + want: []string{ + "Created", + types.ChangeTypeCreate, + types.ChangeTypeRegisterNode, + types.ChangeTypeRunInstances, + types.ChangeTypeUserCreated, + }, + }, + { + pseudo: PseudoUnhealthy, + want: []string{ + types.ChangeTypeBackupFailed, + types.ChangeTypeCertificateExpired, + types.ChangeTypePipelineRunFailed, + types.ChangeTypePlaybookFailed, + "Unhealthy", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.pseudo, func(t *testing.T) { + g := gomega.NewWithT(t) + got, err := ExpandPseudo(tc.pseudo) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.ConsistOf(tc.want)) + }) + } +} + +func TestExpandPseudoUnknown(t *testing.T) { + g := gomega.NewWithT(t) + _, err := ExpandPseudo("@nope") + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(errors.Is(err, ErrUnknownPseudo)).To(gomega.BeTrue()) +} + +func TestExpandChangeTypesMixed(t *testing.T) { + g := gomega.NewWithT(t) + + got, err := expandChangeTypes([]string{ + "PermissionAdded", + "@unhealthy", + "PermissionAdded", // dedupe + "", // skip + }) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + g.Expect(got).To(gomega.HaveKey("PermissionAdded")) + g.Expect(got).To(gomega.HaveKey(types.ChangeTypeBackupFailed)) + g.Expect(got).To(gomega.HaveKey(types.ChangeTypePlaybookFailed)) + g.Expect(got).ToNot(gomega.HaveKey("")) +} + +func TestExpandChangeTypesUnknownPseudoPropagates(t *testing.T) { + g := gomega.NewWithT(t) + + _, err := expandChangeTypes([]string{"Deployment", "@foo"}) + g.Expect(err).To(gomega.HaveOccurred()) + g.Expect(errors.Is(err, ErrUnknownPseudo)).To(gomega.BeTrue()) +} + +func TestExpandChangeTypesEmptyAcceptsAll(t *testing.T) { + g := gomega.NewWithT(t) + + got, err := expandChangeTypes(nil) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.BeEmpty()) + + // Matches helper treats empty set as "accept everything". + r := &GroupingRule{literalChangeTypes: got} + g.Expect(r.Matches("Literally anything")).To(gomega.BeTrue()) +} diff --git a/changegroup/rule.go b/changegroup/rule.go new file mode 100644 index 000000000..1b3b75e26 --- /dev/null +++ b/changegroup/rule.go @@ -0,0 +1,191 @@ +package changegroup + +import ( + "time" + + "github.com/flanksource/duty/types" +) + +// GroupingRule declares how the engine should fold matching config_changes +// into a single change_group. Rules are loaded from ScrapeConfig / +// ScrapePlugin specs (field: spec.transform.changes.grouping) and handed +// to the engine via SetRules. +type GroupingRule struct { + Name string `json:"name" yaml:"name"` + Filter string `json:"filter,omitempty" yaml:"filter,omitempty"` + Scope Scope `json:"scope" yaml:"scope"` + Window Duration `json:"window" yaml:"window"` + CloseAfter Duration `json:"closeAfter,omitempty" yaml:"closeAfter,omitempty"` + ChangeTypes []string `json:"changeTypes,omitempty" yaml:"changeTypes,omitempty"` + Key string `json:"key" yaml:"key"` + Details string `json:"details" yaml:"details"` + Summary string `json:"summary,omitempty" yaml:"summary,omitempty"` + Priority int `json:"priority,omitempty" yaml:"priority,omitempty"` + + // Compiled programs are stashed here at Validate() time so per-change + // evaluation skips re-parsing. Not serialized. + filterProgram Program `json:"-" yaml:"-"` + keyProgram Program `json:"-" yaml:"-"` + detailsProgram Program `json:"-" yaml:"-"` + summaryProgram Program `json:"-" yaml:"-"` + + // literalChangeTypes is the expanded set of literal change_type strings + // this rule matches, after pseudo-type expansion. Populated by Validate. + literalChangeTypes map[string]struct{} `json:"-" yaml:"-"` +} + +// Scope narrows which changes the rule can bind together. +type Scope struct { + Kind string `json:"kind" yaml:"kind"` + Field string `json:"field,omitempty" yaml:"field,omitempty"` + Depth int `json:"depth,omitempty" yaml:"depth,omitempty"` +} + +const ( + ScopeSameConfig = "same_config" + ScopeRelated = "related" + ScopeAll = "all" + ScopeByDetailsField = "by_details_field" +) + +// Duration is a time.Duration that marshals to/from a Go duration string +// (e.g. "5s", "30m") so YAML stays human-readable. +type Duration time.Duration + +func (d Duration) Std() time.Duration { return time.Duration(d) } + +func (d Duration) MarshalJSON() ([]byte, error) { + return []byte(`"` + time.Duration(d).String() + `"`), nil +} + +func (d *Duration) UnmarshalJSON(data []byte) error { + s := string(data) + if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' { + s = s[1 : len(s)-1] + } + if s == "" || s == "null" { + *d = 0 + return nil + } + parsed, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(parsed) + return nil +} + +func (d Duration) MarshalYAML() (any, error) { return time.Duration(d).String(), nil } + +func (d *Duration) UnmarshalYAML(unmarshal func(any) error) error { + var s string + if err := unmarshal(&s); err != nil { + return err + } + if s == "" { + *d = 0 + return nil + } + parsed, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(parsed) + return nil +} + +// ExprKind distinguishes the expected return type of a compiled CEL expression. +type ExprKind int + +const ( + ExprBool ExprKind = iota + ExprString + ExprGroupDetails +) + +// Program is an opaque evaluator-specific compiled expression handle. +type Program any + +// Env is the binding set passed to every evaluator call. +type Env struct { + // Change is the triggering change (one-based map of the change row). + Change map[string]any + // Changes is the full list of members currently in the group, including + // the triggering change (appended last). First eval: len == 1. + Changes []map[string]any + // Group is the current persisted ChangeGroup state or nil on first eval. + Group map[string]any + // Flat contains top-level shortcuts mirroring Change.* for single-change rules. + Flat map[string]any +} + +// Evaluator is implemented by an external CEL-backed evaluator (in config-db). +// duty declares only the interface to avoid pulling in CEL as a dependency. +type Evaluator interface { + EvalBool(prog Program, env Env) (bool, error) + EvalString(prog Program, env Env) (string, error) + EvalGroupDetails(prog Program, env Env) (types.GroupType, error) + Compile(expr string, kind ExprKind) (Program, error) +} + +// Validate compiles every expression on the rule and expands pseudo change +// types. It must be called once when the rule is loaded. Returns an error if +// any expression fails to compile or a pseudo type is unknown. +func (r *GroupingRule) Validate(ev Evaluator) error { + if r.Name == "" { + return ErrEmptyRuleName + } + if r.Details == "" { + return ErrMissingDetails + } + if r.Key == "" { + return ErrMissingKey + } + + literals, err := expandChangeTypes(r.ChangeTypes) + if err != nil { + return err + } + r.literalChangeTypes = literals + + if r.Filter != "" { + p, err := ev.Compile(r.Filter, ExprBool) + if err != nil { + return &CompileError{Rule: r.Name, Field: "filter", Err: err} + } + r.filterProgram = p + } + + p, err := ev.Compile(r.Key, ExprString) + if err != nil { + return &CompileError{Rule: r.Name, Field: "key", Err: err} + } + r.keyProgram = p + + p, err = ev.Compile(r.Details, ExprGroupDetails) + if err != nil { + return &CompileError{Rule: r.Name, Field: "details", Err: err} + } + r.detailsProgram = p + + if r.Summary != "" { + p, err := ev.Compile(r.Summary, ExprString) + if err != nil { + return &CompileError{Rule: r.Name, Field: "summary", Err: err} + } + r.summaryProgram = p + } + + return nil +} + +// Matches reports whether the rule's change_types accept the given literal +// change_type string. An empty ChangeTypes list matches everything (after the +// filter expression is also applied). +func (r *GroupingRule) Matches(changeType string) bool { + if len(r.literalChangeTypes) == 0 { + return true + } + _, ok := r.literalChangeTypes[changeType] + return ok +} diff --git a/changegroup/upsert.go b/changegroup/upsert.go new file mode 100644 index 000000000..c72a7f27f --- /dev/null +++ b/changegroup/upsert.go @@ -0,0 +1,197 @@ +package changegroup + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "hash/fnv" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" +) + +// hashKey combines the rule name and raw key expression output into the +// final stored correlation_key. Including the rule name prevents accidental +// collisions between two rules whose key expressions happen to produce the +// same string. +func hashKey(ruleName, rawKey string) string { + h := sha256.Sum256([]byte(ruleName + "|" + rawKey)) + return ruleName + ":" + hex.EncodeToString(h[:]) +} + +// advisoryLockKey converts a (type, correlation_key) pair into an int64 +// suitable for pg_advisory_xact_lock. The lock serializes concurrent upserts +// for the same logical group across workers. +func advisoryLockKey(groupType, correlationKey string) int64 { + h := fnv.New64a() + h.Write([]byte(groupType)) + h.Write([]byte{0}) + h.Write([]byte(correlationKey)) + return int64(h.Sum64()) //nolint:gosec // wraparound is fine for advisory locks +} + +// upsertAndAttach finds-or-creates an open group for the given correlation +// key, evaluates Details and Summary against the full member list (including +// the new change), merges the result into stored details, and sets +// change.GroupID. Runs in a single transaction with an advisory lock. +func (e *Engine) upsertAndAttach( + ctx context.Context, + rule *GroupingRule, + correlationKey string, + change *models.ConfigChange, +) error { + return ctx.DB().Transaction(func(tx *gorm.DB) error { + // Placeholder group type used only for the advisory lock key while we + // look up or create the real row. We don't know the final group type + // until we evaluate Details, so lock on (rule.Name, correlationKey) — + // rule name is already baked into correlationKey, so locking on that + // alone is sufficient. + lockKey := advisoryLockKey(rule.Name, correlationKey) + if err := tx.Exec(`SELECT pg_advisory_xact_lock(?)`, lockKey).Error; err != nil { + return fmt.Errorf("advisory lock: %w", err) + } + + // Find existing open group for this correlation_key, regardless of type. + // The unique index is (type, correlation_key) WHERE status='open', but + // rule name + hash in the key already makes collisions across types + // astronomically unlikely. + var existing models.ChangeGroup + err := tx.Where("correlation_key = ? AND status = ?", correlationKey, models.ChangeGroupStatusOpen). + Take(&existing).Error + + var members []models.ConfigChange + var currentGroup *models.ChangeGroup + if err == nil { + currentGroup = &existing + if err := tx.Where("group_id = ?", existing.ID). + Order("created_at ASC, id ASC"). + Find(&members).Error; err != nil { + return fmt.Errorf("load group members: %w", err) + } + } else if err != gorm.ErrRecordNotFound { + return fmt.Errorf("lookup open group: %w", err) + } + + // Append the triggering change as the last member when computing env. + memberMaps := make([]map[string]any, 0, len(members)+1) + for i := range members { + memberMaps = append(memberMaps, changeAsMap(&members[i])) + } + memberMaps = append(memberMaps, changeAsMap(change)) + + env := Env{ + Change: changeAsMap(change), + Changes: memberMaps, + Flat: changeAsMap(change), + } + if currentGroup != nil { + env.Group = groupAsMap(currentGroup) + } + + // Details is required. + incomingDetails, err := e.evaluator.EvalGroupDetails(rule.detailsProgram, env) + if err != nil { + return &EvalError{Rule: rule.Name, Field: "details", Err: err} + } + if incomingDetails == nil { + return fmt.Errorf("changegroup: rule %q details evaluated to nil", rule.Name) + } + + summary := "" + if rule.summaryProgram != nil { + summary, err = e.evaluator.EvalString(rule.summaryProgram, env) + if err != nil { + return &EvalError{Rule: rule.Name, Field: "summary", Err: err} + } + } + + // Determine the effective group for this attach. + if currentGroup == nil { + createdAt := time.Now().UTC() + if change.CreatedAt != nil { + createdAt = *change.CreatedAt + } + newGroup := models.ChangeGroup{ + ID: uuid.New(), + Type: incomingDetails.Kind(), + Summary: summary, + CorrelationKey: correlationKey, + Source: models.ChangeGroupSourceRule + ":" + rule.Name, + RuleName: &rule.Name, + Status: models.ChangeGroupStatusOpen, + StartedAt: createdAt, + LastMemberAt: createdAt, + MemberCount: 0, // trigger will bump on attach + CreatedAt: createdAt, + UpdatedAt: createdAt, + } + raw, err := json.Marshal(incomingDetails) + if err != nil { + return fmt.Errorf("marshal group details: %w", err) + } + newGroup.Details = types.JSON(raw) + if err := tx.Create(&newGroup).Error; err != nil { + return fmt.Errorf("create change_group: %w", err) + } + currentGroup = &newGroup + } else { + stored, err := currentGroup.TypedDetails() + if err != nil { + return fmt.Errorf("unmarshal stored group details: %w", err) + } + merged, err := Merge(stored, incomingDetails) + if err != nil { + return fmt.Errorf("merge group details: %w", err) + } + raw, err := json.Marshal(merged) + if err != nil { + return fmt.Errorf("marshal merged details: %w", err) + } + updates := map[string]any{ + "details": types.JSON(raw), + "updated_at": time.Now().UTC(), + } + if summary != "" { + updates["summary"] = summary + } + if err := tx.Model(&models.ChangeGroup{}). + Where("id = ?", currentGroup.ID). + Updates(updates).Error; err != nil { + return fmt.Errorf("update change_group: %w", err) + } + } + + // Attach the change to the group. The 047 trigger maintains + // member_count / last_member_at / started_at. + change.GroupID = ¤tGroup.ID + if err := tx.Model(&models.ConfigChange{}). + Where("id = ?", change.ID). + Update("group_id", currentGroup.ID).Error; err != nil { + return fmt.Errorf("attach change to group: %w", err) + } + return nil + }) +} + +// groupAsMap projects a ChangeGroup into the CEL binding shape. +func groupAsMap(g *models.ChangeGroup) map[string]any { + return map[string]any{ + "id": g.ID.String(), + "type": g.Type, + "summary": g.Summary, + "correlation_key": g.CorrelationKey, + "source": g.Source, + "status": g.Status, + "started_at": g.StartedAt, + "ended_at": g.EndedAt, + "last_member_at": g.LastMemberAt, + "member_count": g.MemberCount, + "details": g.Details, + } +} diff --git a/connection/azure.go b/connection/azure.go index 89b5e4a6c..f9d6f2488 100644 --- a/connection/azure.go +++ b/connection/azure.go @@ -1,7 +1,11 @@ package connection import ( + "context" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" @@ -13,6 +17,11 @@ type AzureConnection struct { ClientID *types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"` ClientSecret *types.EnvVar `yaml:"clientSecret,omitempty" json:"clientSecret,omitempty"` TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty"` + + // bearerToken is populated from the referenced connection's + // Properties["bearer"] when the connection has no Username/Password set. + // It is an unexported fallback — not part of the inline YAML surface. + bearerToken string `json:"-" yaml:"-"` } // HydrateConnection attempts to find the connection by name @@ -33,6 +42,10 @@ func (g *AzureConnection) HydrateConnection(ctx ConnectionContext) error { if g.TenantID == "" { g.TenantID = connection.Properties["tenant"] } + + if connection.Username == "" && connection.Password == "" { + g.bearerToken = connection.Properties["bearer"] + } } return nil @@ -45,6 +58,9 @@ func (g *AzureConnection) FromModel(connection models.Connection) { if tenantID, ok := connection.Properties["tenant"]; ok { g.TenantID = tenantID } + if connection.Username == "" && connection.Password == "" { + g.bearerToken = connection.Properties["bearer"] + } } func (g AzureConnection) ToModel() models.Connection { @@ -60,5 +76,16 @@ func (g AzureConnection) ToModel() models.Connection { } func (g *AzureConnection) TokenCredential() (azcore.TokenCredential, error) { + if (g.ClientID == nil || g.ClientID.IsEmpty()) && + (g.ClientSecret == nil || g.ClientSecret.IsEmpty()) && + g.bearerToken != "" { + return staticTokenCredential{token: g.bearerToken}, nil + } return azidentity.NewClientSecretCredential(g.TenantID, g.ClientID.String(), g.ClientSecret.String(), nil) } + +type staticTokenCredential struct{ token string } + +func (s staticTokenCredential) GetToken(_ context.Context, _ policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: s.token, ExpiresOn: time.Now().Add(time.Hour)}, nil +} diff --git a/connection/merge_test.go b/connection/merge_test.go index a57a47815..501d14772 100644 --- a/connection/merge_test.go +++ b/connection/merge_test.go @@ -5,6 +5,8 @@ import ( "strconv" "testing" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" "github.com/onsi/gomega" @@ -524,6 +526,58 @@ func TestAzureConnectionFallsBackToConnection(t *testing.T) { g.Expect(conn.TenantID).To(gomega.Equal("conn-tenant")) } +func TestAzureConnectionUsesBearerWhenClientSecretAbsent(t *testing.T) { + g := gomega.NewWithT(t) + + azConn := &models.Connection{ + Type: models.ConnectionTypeAzure, + Properties: types.JSONStringMap{ + "bearer": "test-bearer-token", + }, + } + ctx := mockConnectionContext{Context: gocontext.Background(), connection: azConn} + + conn := AzureConnection{ConnectionName: "connection://azure"} + err := conn.HydrateConnection(ctx) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(conn.ClientID.ValueStatic).To(gomega.BeEmpty()) + g.Expect(conn.ClientSecret.ValueStatic).To(gomega.BeEmpty()) + + cred, err := conn.TokenCredential() + g.Expect(err).ToNot(gomega.HaveOccurred()) + token, err := cred.GetToken(gocontext.Background(), policy.TokenRequestOptions{}) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(token.Token).To(gomega.Equal("test-bearer-token")) +} + +func TestAzureConnectionPrefersClientSecretOverBearer(t *testing.T) { + g := gomega.NewWithT(t) + + azConn := &models.Connection{ + Type: models.ConnectionTypeAzure, + Username: "conn-client-id", + Password: "conn-client-secret", + Properties: types.JSONStringMap{ + "tenant": "conn-tenant", + "bearer": "should-be-ignored", + }, + } + ctx := mockConnectionContext{Context: gocontext.Background(), connection: azConn} + + conn := AzureConnection{ConnectionName: "connection://azure"} + err := conn.HydrateConnection(ctx) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(conn.ClientID.ValueStatic).To(gomega.Equal("conn-client-id")) + g.Expect(conn.ClientSecret.ValueStatic).To(gomega.Equal("conn-client-secret")) + + cred, err := conn.TokenCredential() + g.Expect(err).ToNot(gomega.HaveOccurred()) + _, isStatic := cred.(staticTokenCredential) + g.Expect(isStatic).To(gomega.BeFalse(), "should return ClientSecretCredential, not bearer") + _, isClientSecret := cred.(*azidentity.ClientSecretCredential) + g.Expect(isClientSecret).To(gomega.BeTrue()) +} + func TestOpensearchConnectionInlineOverridesConnection(t *testing.T) { g := gomega.NewWithT(t) diff --git a/context/properties.go b/context/properties.go index d0b09bdbb..abf525be9 100644 --- a/context/properties.go +++ b/context/properties.go @@ -195,6 +195,46 @@ func (h HierarchicalProperties) getProperty(key string) (string, bool) { return v, ok } +// WithPrefix returns all properties whose key begins with prefix, resolved +// with the normal precedence (CLI/env → local → parent chain → global DB). +// The returned keys have the prefix STRIPPED. Later sources in the chain +// override earlier ones, matching getProperty's semantics. +func (h HierarchicalProperties) WithPrefix(prefix string) map[string]string { + out := make(map[string]string) + + // Walk global first (lowest precedence). + for k, v := range h.global { + if after, ok := strings.CutPrefix(k, prefix); ok { + out[after] = v + } + } + + // Then walk the parent chain from root to this node (so this node + // overrides its ancestors). + var chain []HierarchicalProperties + cur := &h + for cur != nil { + chain = append(chain, *cur) + cur = cur.parent + } + for i := len(chain) - 1; i >= 0; i-- { + for k, v := range chain[i].local { + if after, ok := strings.CutPrefix(k, prefix); ok { + out[after] = v + } + } + } + + // Finally, CLI/env takes highest precedence. + for k, v := range properties.Global.GetAll() { + if after, ok := strings.CutPrefix(k, prefix); ok { + out[after] = v + } + } + + return out +} + func (k Context) globalProperties() Properties { if val, ok := propertyCache.Get("global"); ok { return val.(map[string]string) diff --git a/db.go b/db.go index 4c9c59db3..fc1163c62 100644 --- a/db.go +++ b/db.go @@ -5,10 +5,13 @@ import ( "database/sql" "flag" "fmt" + "sort" + "strings" "time" "github.com/exaring/otelpgx" "github.com/flanksource/commons/logger" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/stdlib" gormpostgres "gorm.io/driver/postgres" @@ -78,6 +81,84 @@ func NewGorm(connection string, config *gorm.Config) (*gorm.DB, error) { return Gorm, nil } +// NewGormFromPool creates a Gorm DB that reuses an existing *pgxpool.Pool. +// This is the preferred path for the main application: it shares the same +// connection pool (and therefore the ConnConfig.OnNotice handler, pgx +// Tracer, and MaxConns) with any direct pgxpool users on the same context. +// +// In particular, without sharing the pool, RAISE NOTICE / RAISE WARNING +// messages emitted by server-side functions are invisible to Go callers +// using ctx.DB() (GORM) because the stdlib driver creates its own pgx +// config with no notice handler. +// +// stdlib.OpenDBFromPool automatically sets db.SetMaxIdleConns(0), so GORM +// will not hoard connections from the pool. +func NewGormFromPool(pool *pgxpool.Pool, config *gorm.Config) (*gorm.DB, error) { + db := stdlib.OpenDBFromPool(pool) + + gormDB, err := gorm.Open( + gormpostgres.New(gormpostgres.Config{Conn: db}), + config, + ) + if err != nil { + return nil, err + } + + if err := gormDB.Use(tracing.NewPlugin()); err != nil { + return nil, fmt.Errorf("error setting up tracing: %w", err) + } + + return gormDB, nil +} + +// SessionPropertyPrefix is the property-name prefix used to route values +// into Postgres session/transaction-local settings. A property +// "postgres.session.debug_log.enabled=on" translates to +// `SET LOCAL debug_log.enabled = 'on'` inside the provided transaction. +const SessionPropertyPrefix = "postgres.session." + +// ApplySessionProperties runs `SET LOCAL = ''` inside the +// supplied GORM transaction for every property in `ctx.Properties()` whose +// key starts with SessionPropertyPrefix. The prefix is stripped before the +// SET is issued. Values are passed as text; Postgres will coerce them as +// needed by the GUC's type. +// +// The transaction MUST be an open tx (returned by `db.Begin()` or +// `db.WithContext(...).Begin()`); `SET LOCAL` outside a tx has no effect. +// +// Typical usage: +// +// tx := ctx.DB().Begin() +// if err := duty.ApplySessionProperties(ctx, tx); err != nil { ... } +// defer tx.Rollback() // or commit +// ... +func ApplySessionProperties(ctx dutyContext.Context, tx *gorm.DB) error { + if tx == nil { + return fmt.Errorf("ApplySessionProperties: nil transaction") + } + settings := ctx.Properties().WithPrefix(SessionPropertyPrefix) + if len(settings) == 0 { + return nil + } + // Sort keys for deterministic apply order (helps tests and log output). + keys := make([]string, 0, len(settings)) + for k := range settings { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := settings[k] + // set_config(setting, new_value, is_local) is the parameterizable + // equivalent of `SET LOCAL = ` — it accepts the GUC + // name as a text parameter, avoiding any need to escape/quote the + // identifier ourselves. + if err := tx.Exec("SELECT set_config(?, ?, true)", k, v).Error; err != nil { + return fmt.Errorf("failed to set session property %q=%q: %w", k, v, err) + } + } + return nil +} + func getConnection(connection string) (string, error) { pgxConfig, err := drivers.ParseURL(connection) if err != nil { @@ -121,6 +202,28 @@ func NewPgxPool(connection string) (*pgxpool.Pool, error) { }), ) + // Route Postgres NOTICE / WARNING messages (emitted via `RAISE NOTICE` + // or `RAISE WARNING` from server-side functions) to the application + // logger so server-side debug output is visible without needing to + // attach psql. Severity → log level mapping mirrors Postgres semantics. + config.ConnConfig.OnNotice = func(_ *pgconn.PgConn, n *pgconn.Notice) { + if n == nil { + return + } + switch strings.ToUpper(n.Severity) { + case "ERROR", "FATAL", "PANIC": + logger.Errorf("pg %s: %s", n.Severity, n.Message) + case "WARNING": + logger.Warnf("pg %s: %s", n.Severity, n.Message) + case "NOTICE", "INFO": + logger.Infof("pg %s: %s", n.Severity, n.Message) + case "LOG", "DEBUG": + logger.Debugf("pg %s: %s", n.Severity, n.Message) + default: + logger.Infof("pg %s: %s", n.Severity, n.Message) + } + } + // prevent deadlocks from concurrent queries if config.MaxConns < 20 { config.MaxConns = 20 @@ -217,7 +320,10 @@ func SetupDB(config api.Config) (gormDB *gorm.DB, pgxpool *pgxpool.Pool, err err cfg.Logger = dutyGorm.NewSqlLogger(logger.GetLogger(config.LogName)) } - gormDB, err = NewGorm(config.ConnectionString, cfg) + // Share the pgxpool with GORM so Notice/Warning messages emitted by + // server-side functions (RAISE NOTICE / RAISE WARNING) flow into the + // pool's ConnConfig.OnNotice handler installed by NewPgxPool. + gormDB, err = NewGormFromPool(pgxpool, cfg) if err != nil { return } diff --git a/hack/generate-schemas/main.go b/hack/generate-schemas/main.go index 721c14640..e68faefda 100644 --- a/hack/generate-schemas/main.go +++ b/hack/generate-schemas/main.go @@ -1,6 +1,8 @@ package main import ( + "encoding/json" + "fmt" "os" "path" @@ -10,21 +12,52 @@ import ( "github.com/spf13/cobra" ) -var schemas = map[string]any{ +var generatedSchemas = map[string]any{ "resource_selector": &types.ResourceSelector{}, "resource_selectors": &[]types.ResourceSelector{}, } +// change-types is maintained by hand because the reflective schema generator +// does not model the kind-discriminated union shape correctly. +var handwrittenSchemas = map[string]string{ + "change-types": "change-types.handwritten.schema.json", +} + +func writeHandwrittenSchema(dst, src string) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("unable to read handwritten schema %s: %w", src, err) + } + + if !json.Valid(data) { + return fmt.Errorf("handwritten schema %s is not valid json", src) + } + + if err := os.WriteFile(dst, data, 0644); err != nil { + return fmt.Errorf("unable to write handwritten schema to %s: %w", dst, err) + } + + return nil +} + var generateSchema = &cobra.Command{ Use: "generate-schema", Run: func(cmd *cobra.Command, args []string) { - for file, obj := range schemas { + for file, obj := range generatedSchemas { p := path.Join("../../schema/openapi", file+".schema.json") if err := openapi.WriteSchemaToFile(p, obj); err != nil { logger.Fatalf("unable to save schema: %v", err) } logger.Infof("Saved OpenAPI schema to %s", p) } + + for file, src := range handwrittenSchemas { + dst := path.Join("../../schema/openapi", file+".schema.json") + if err := writeHandwrittenSchema(dst, src); err != nil { + logger.Fatalf("unable to save handwritten schema: %v", err) + } + logger.Infof("Saved OpenAPI schema to %s", dst) + } }, } diff --git a/models/change_group.go b/models/change_group.go new file mode 100644 index 000000000..f968895b9 --- /dev/null +++ b/models/change_group.go @@ -0,0 +1,50 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + + "github.com/flanksource/duty/types" +) + +// ChangeGroup represents the change_groups database table — a logical +// grouping of correlated config_changes rows. +type ChangeGroup struct { + ID uuid.UUID `gorm:"primaryKey;column:id;default:generate_ulid()" json:"id"` + Type string `gorm:"column:type" json:"type"` + Summary string `gorm:"column:summary" json:"summary"` + CorrelationKey string `gorm:"column:correlation_key" json:"correlation_key"` + Source string `gorm:"column:source" json:"source"` + RuleName *string `gorm:"column:rule_name" json:"rule_name,omitempty"` + Status string `gorm:"column:status" json:"status"` + StartedAt time.Time `gorm:"column:started_at" json:"started_at"` + EndedAt *time.Time `gorm:"column:ended_at" json:"ended_at,omitempty"` + LastMemberAt time.Time `gorm:"column:last_member_at" json:"last_member_at"` + MemberCount int `gorm:"column:member_count" json:"member_count"` + Details types.JSON `gorm:"column:details" json:"details,omitempty"` + CreatedAt time.Time `gorm:"column:created_at" json:"created_at"` + UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at"` +} + +const ( + ChangeGroupStatusOpen = "open" + ChangeGroupStatusClosed = "closed" + + ChangeGroupSourceExplicit = "explicit" + ChangeGroupSourceRule = "rule" +) + +func (ChangeGroup) TableName() string { return "change_groups" } + +func (g ChangeGroup) PK() string { return g.ID.String() } + +// TypedDetails returns the strongly-typed GroupType for the Details column +// by inspecting the "kind" envelope. +func (g ChangeGroup) TypedDetails() (types.GroupType, error) { + if len(g.Details) == 0 { + return nil, nil + } + return types.UnmarshalGroupDetails(json.RawMessage(g.Details)) +} diff --git a/models/changes.go b/models/changes.go index 686d6e61a..73c9dab2e 100644 --- a/models/changes.go +++ b/models/changes.go @@ -31,6 +31,7 @@ type CatalogChange struct { AgentID *uuid.UUID `gorm:"column:agent_id" json:"agent_id"` Path string `gorm:"column:path" json:"path"` InsertedAt *time.Time `gorm:"column:inserted_at" json:"inserted_at,omitempty"` + GroupID *uuid.UUID `gorm:"column:group_id" json:"group_id,omitempty"` } func (c CatalogChange) GetID() string { diff --git a/models/config.go b/models/config.go index 20f37e3d5..81ec0ca33 100644 --- a/models/config.go +++ b/models/config.go @@ -720,6 +720,14 @@ type ConfigChange struct { // Artifacts associated with this change (screenshots, HAR files, etc.) Artifacts []Artifact `gorm:"foreignKey:ConfigChangeID" json:"artifacts,omitempty"` + + // Action is the resolved change mapping action (move-up, copy-up, copy, move, delete, ignore). + // Not stored in the database — populated by the scrape pipeline for diagnostics. + Action string `gorm:"-" json:"action,omitempty"` + + // GroupID is the optional logical grouping this change belongs to. + // Set explicitly by producers or by the grouping engine. + GroupID *uuid.UUID `gorm:"column:group_id" json:"group_id,omitempty"` } func (c ConfigChange) Pretty() api.Text { diff --git a/query/change_groups.go b/query/change_groups.go new file mode 100644 index 000000000..c4db47b9a --- /dev/null +++ b/query/change_groups.go @@ -0,0 +1,150 @@ +package query + +import ( + "strings" + "time" + + "github.com/google/uuid" + + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" +) + +// ChangeGroupsSearchRequest describes filters for FindChangeGroups. +type ChangeGroupsSearchRequest struct { + // Type is a comma-separated list of change_group.type values. + Type string `query:"type" json:"type,omitempty"` + // Status is open|closed. Empty means any. + Status string `query:"status" json:"status,omitempty"` + // ConfigID restricts to groups that have at least one member belonging to this config_id. + ConfigID *uuid.UUID `query:"config_id" json:"config_id,omitempty"` + // Summary is a case-insensitive substring search on the summary column. + Summary string `query:"summary" json:"summary,omitempty"` + // Since / Until bound started_at. + Since *time.Time `query:"since" json:"since,omitempty"` + Until *time.Time `query:"until" json:"until,omitempty"` + // Pagination. + Page int `query:"page" json:"page,omitempty"` + PageSize int `query:"page_size" json:"page_size,omitempty"` +} + +func (r *ChangeGroupsSearchRequest) setDefaults() { + if r.Page < 1 { + r.Page = 1 + } + if r.PageSize <= 0 || r.PageSize > 500 { + r.PageSize = 50 + } +} + +// FindChangeGroups returns change_groups matching the search request. +func FindChangeGroups(ctx context.Context, req ChangeGroupsSearchRequest) ([]models.ChangeGroup, error) { + req.setDefaults() + + q := ctx.DB().Model(&models.ChangeGroup{}) + + if req.Type != "" { + q = q.Where("type IN ?", strings.Split(req.Type, ",")) + } + if req.Status != "" { + q = q.Where("status = ?", req.Status) + } + if req.Summary != "" { + q = q.Where("summary ILIKE ?", "%"+req.Summary+"%") + } + if req.Since != nil { + q = q.Where("started_at >= ?", *req.Since) + } + if req.Until != nil { + q = q.Where("started_at <= ?", *req.Until) + } + if req.ConfigID != nil { + q = q.Where( + "id IN (SELECT group_id FROM config_changes WHERE config_id = ? AND group_id IS NOT NULL)", + *req.ConfigID, + ) + } + + q = q.Order("started_at DESC"). + Limit(req.PageSize). + Offset((req.Page - 1) * req.PageSize) + + var groups []models.ChangeGroup + if err := q.Find(&groups).Error; err != nil { + return nil, err + } + return groups, nil +} + +// GetChangeGroup loads a single change_group by id. +func GetChangeGroup(ctx context.Context, id uuid.UUID) (*models.ChangeGroup, error) { + var g models.ChangeGroup + if err := ctx.DB().Where("id = ?", id).Take(&g).Error; err != nil { + return nil, err + } + return &g, nil +} + +// GetGroupMembers returns the config_changes rows belonging to the given group. +func GetGroupMembers(ctx context.Context, id uuid.UUID) ([]models.ConfigChange, error) { + var members []models.ConfigChange + if err := ctx.DB(). + Where("group_id = ?", id). + Order("created_at ASC, id ASC"). + Find(&members).Error; err != nil { + return nil, err + } + return members, nil +} + +// ChangeGroupSummary is a row from the change_groups_summary view. +type ChangeGroupSummary struct { + ID uuid.UUID `gorm:"column:id" json:"id"` + Type string `gorm:"column:type" json:"type"` + Summary string `gorm:"column:summary" json:"summary"` + Source string `gorm:"column:source" json:"source"` + RuleName *string `gorm:"column:rule_name" json:"rule_name,omitempty"` + Status string `gorm:"column:status" json:"status"` + StartedAt time.Time `gorm:"column:started_at" json:"started_at"` + EndedAt *time.Time `gorm:"column:ended_at" json:"ended_at,omitempty"` + LastMemberAt time.Time `gorm:"column:last_member_at" json:"last_member_at"` + MemberCount int `gorm:"column:member_count" json:"member_count"` + DistinctConfigCount int `gorm:"column:distinct_config_count" json:"distinct_config_count"` + DurationSeconds float64 `gorm:"column:duration_seconds" json:"duration_seconds"` +} + +func (ChangeGroupSummary) TableName() string { return "change_groups_summary" } + +// FindChangeGroupsSummary returns aggregated rows from change_groups_summary, +// optionally filtered. Filters match FindChangeGroups where applicable. +func FindChangeGroupsSummary(ctx context.Context, req ChangeGroupsSearchRequest) ([]ChangeGroupSummary, error) { + req.setDefaults() + + q := ctx.DB().Table("change_groups_summary") + + if req.Type != "" { + q = q.Where("type IN ?", strings.Split(req.Type, ",")) + } + if req.Status != "" { + q = q.Where("status = ?", req.Status) + } + if req.Summary != "" { + q = q.Where("summary ILIKE ?", "%"+req.Summary+"%") + } + if req.Since != nil { + q = q.Where("started_at >= ?", *req.Since) + } + if req.Until != nil { + q = q.Where("started_at <= ?", *req.Until) + } + + q = q.Order("started_at DESC"). + Limit(req.PageSize). + Offset((req.Page - 1) * req.PageSize) + + var out []ChangeGroupSummary + if err := q.Scan(&out).Error; err != nil { + return nil, err + } + return out, nil +} diff --git a/query/config_changes.go b/query/config_changes.go index c329cc9c5..86d4ca5b5 100644 --- a/query/config_changes.go +++ b/query/config_changes.go @@ -37,6 +37,13 @@ type CatalogChangesSearchRequest struct { Summary string `query:"summary" json:"summary"` Source string `query:"source" json:"source"` + // GroupID restricts results to changes that belong to a specific change_group. + GroupID *uuid.UUID `query:"group_id" json:"group_id,omitempty"` + // GroupType is a comma-separated list of change_group.type values. + GroupType string `query:"group_type" json:"group_type,omitempty"` + // Grouped is a tri-state filter: nil = any, true = only grouped, false = only ungrouped. + Grouped *bool `query:"grouped" json:"grouped,omitempty"` + createdBy *uuid.UUID externalCreatedBy string @@ -295,6 +302,26 @@ func FindCatalogChanges(ctx context.Context, req CatalogChangesSearchRequest) (r } } + if req.GroupID != nil { + clauses = append(clauses, clause.Eq{Column: clause.Column{Name: "group_id"}, Value: *req.GroupID}) + } + + if req.Grouped != nil { + if *req.Grouped { + dbQuery = dbQuery.Where("group_id IS NOT NULL") + } else { + dbQuery = dbQuery.Where("group_id IS NULL") + } + } + + if req.GroupType != "" { + groupTypes := strings.Split(req.GroupType, ",") + dbQuery = dbQuery.Where( + "group_id IN (SELECT id FROM change_groups WHERE type IN ?)", + groupTypes, + ) + } + // Determine table: single UUID uses related_changes_recursive, multi-ID or query uses IN clause table := dbQuery.Table("catalog_changes") if len(configIDs) == 1 { diff --git a/query/config_tree.go b/query/config_tree.go index d371a19e0..bd11e1d45 100644 --- a/query/config_tree.go +++ b/query/config_tree.go @@ -180,9 +180,15 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi parentID := lo.FromPtr(c.ParentID) if parent, ok := nodes[parentID]; ok { parent.children = append(parent.children, nodes[c.ID]) - } else { - targetNode.children = append(targetNode.children, nodes[c.ID]) + continue + } + if c.Path != "" { + if parent := findNearestAncestor(c.Path, nodes); parent != nil { + parent.children = append(parent.children, nodes[c.ID]) + continue + } } + targetNode.children = append(targetNode.children, nodes[c.ID]) } parentIDs := make(map[uuid.UUID]bool, len(parents)) @@ -211,17 +217,9 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi wired[rc.ID] = true node := nodes[rc.ID] if rc.Path != "" { - segments := strings.Split(rc.Path, ".") - // Last segment is the node's own ID (SetParent appends ci.ID), so use penultimate - if len(segments) >= 2 { - if parentStr := segments[len(segments)-2]; parentStr != "" { - if pid, err := uuid.Parse(parentStr); err == nil { - if parent, ok := nodes[pid]; ok && parent != node && !parentIDs[pid] { - parent.children = append(parent.children, node) - continue - } - } - } + if parent := findNearestAncestor(rc.Path, nodes); parent != nil && parent != node && !parentIDs[parent.ID] { + parent.children = append(parent.children, node) + continue } } targetNode.children = append(targetNode.children, node) @@ -237,6 +235,18 @@ func buildConfigTree(config *models.ConfigItem, parents []models.ConfigItem, chi return toConfigTreeNode(root, make(map[*ptrNode]bool)) } +func findNearestAncestor(path string, nodes map[uuid.UUID]*ptrNode) *ptrNode { + segments := strings.Split(path, ".") + for i := len(segments) - 1; i >= 0; i-- { + if pid, err := uuid.Parse(segments[i]); err == nil { + if parent, ok := nodes[pid]; ok { + return parent + } + } + } + return nil +} + func toConfigTreeNode(n *ptrNode, visited map[*ptrNode]bool) *ConfigTreeNode { result := &ConfigTreeNode{ ConfigItem: n.ConfigItem, diff --git a/query/query_logger.go b/query/query_logger.go index a30ba3591..0eb3d95d3 100644 --- a/query/query_logger.go +++ b/query/query_logger.go @@ -3,7 +3,9 @@ package query import ( "fmt" "reflect" + "slices" "strings" + "sync" "time" clickyapi "github.com/flanksource/clicky/api" @@ -11,16 +13,61 @@ import ( "github.com/flanksource/duty/context" ) +type QueryLogEntry struct { + Name string `json:"name"` + Args string `json:"args,omitempty"` + Count int `json:"count"` + Duration int64 `json:"duration"` + Error string `json:"error,omitempty"` + Summary string `json:"summary,omitempty"` + Pretty string `json:"pretty"` +} + +type QueryLog struct { + mu sync.Mutex + entries []QueryLogEntry +} + +func (q *QueryLog) Append(e QueryLogEntry) { + q.mu.Lock() + defer q.mu.Unlock() + q.entries = append(q.entries, e) +} + +func (q *QueryLog) Entries() []QueryLogEntry { + q.mu.Lock() + defer q.mu.Unlock() + return slices.Clone(q.entries) +} + +type queryLogKey struct{} + +func WithQueryLog(ctx context.Context) (context.Context, *QueryLog) { + log := &QueryLog{} + return ctx.WithValue(queryLogKey{}, log), log +} + +func GetQueryLog(ctx context.Context) *QueryLog { + if v, ok := ctx.Value(queryLogKey{}).(*QueryLog); ok { + return v + } + return nil +} + type QueryLogger struct { - logger logger.Verbose + logger logger.Verbose + queryLog *QueryLog } type QueryTimer struct { - logger logger.Verbose - label clickyapi.Text - start time.Time - results any - ended bool + logger logger.Verbose + queryLog *QueryLog + name string + args string + label clickyapi.Text + start time.Time + results any + ended bool } func NewQueryLogger(ctx context.Context) QueryLogger { @@ -28,14 +75,16 @@ func NewQueryLogger(ctx context.Context) QueryLogger { if ctx.Properties().On(false, "query.log") { l = ctx.Logger.V(0) } - return QueryLogger{logger: l} + return QueryLogger{logger: l, queryLog: GetQueryLog(ctx)} } func (q QueryLogger) Start(entity string) *QueryTimer { return &QueryTimer{ - logger: q.logger, - label: clickyapi.Text{Content: "[" + entity + "]", Style: "text-blue-600 font-bold"}, - start: time.Now(), + logger: q.logger, + queryLog: q.queryLog, + name: entity, + label: clickyapi.Text{Content: "[" + entity + "]", Style: "text-blue-600 font-bold"}, + start: time.Now(), } } @@ -46,6 +95,10 @@ func (t *QueryTimer) Arg(key string, value any) *QueryTimer { } t.label = t.label.AddText(fmt.Sprintf(" %s=", key), "text-gray-500"). AddText(s) + if t.args != "" { + t.args += " " + } + t.args += fmt.Sprintf("%s=%s", key, s) return t } @@ -59,15 +112,23 @@ func (t *QueryTimer) End(err *error) { return } t.ended = true - if !t.logger.Enabled() { - return - } elapsed := time.Since(t.start) + + var entry QueryLogEntry + if t.queryLog != nil { + entry.Name = t.name + entry.Args = t.args + entry.Duration = elapsed.Milliseconds() + } + label := t.label.AddText(" => ", "text-gray-400") if err != nil && *err != nil { label = label.AddText(fmt.Sprintf("error: %v", *err), "text-red-600") + if t.queryLog != nil { + entry.Error = (*err).Error() + } } else if t.results != nil { count := sliceLen(t.results) countStyle := "text-green-600" @@ -75,13 +136,26 @@ func (t *QueryTimer) End(err *error) { countStyle = "text-red-600" } label = label.AddText(fmt.Sprintf("%d", count), countStyle) - label = label.AddText(summaryText(t.results, count), "text-gray-400") + summary := summaryText(t.results, count) + label = label.AddText(summary, "text-gray-400") + if t.queryLog != nil { + entry.Count = count + entry.Summary = summary + } } else { label = label.AddText("timed out", "text-yellow-600") } label = label.AddText(fmt.Sprintf(" in %dms", elapsed.Milliseconds()), "text-gray-400") - t.logger.Infof("%s", label.ANSI()) + + if t.queryLog != nil { + entry.Pretty = label.String() + t.queryLog.Append(entry) + } + + if t.logger.Enabled() { + t.logger.Infof("%s", label.ANSI()) + } } func sliceLen(v any) int { diff --git a/query/resource_selector.go b/query/resource_selector.go index cc1146d29..36ff0af4e 100644 --- a/query/resource_selector.go +++ b/query/resource_selector.go @@ -691,14 +691,16 @@ func queryTableWithResourceSelectors( var output []uuid.UUID for _, resourceSelector := range resourceSelectors { - items, err := queryResourceSelector[uuid.UUID](ctx, limit, []string{"id"}, resourceSelector, table) - if err != nil { - return nil, err - } + for _, expanded := range resourceSelector.Expand() { + items, err := queryResourceSelector[uuid.UUID](ctx, limit, []string{"id"}, expanded, table) + if err != nil { + return nil, err + } - output = append(output, items...) - if limit > 0 && len(output) >= limit { - return output[:limit], nil + output = append(output, items...) + if limit > 0 && len(output) >= limit { + return output[:limit], nil + } } } @@ -716,14 +718,16 @@ func QueryTableColumnsWithResourceSelectors[T any]( var output []T for _, resourceSelector := range resourceSelectors { - items, err := queryResourceSelector[T](ctx, limit, selectColumns, resourceSelector, table, clauses...) - if err != nil { - return nil, err - } + for _, expanded := range resourceSelector.Expand() { + items, err := queryResourceSelector[T](ctx, limit, selectColumns, expanded, table, clauses...) + if err != nil { + return nil, err + } - output = append(output, items...) - if limit > 0 && len(output) >= limit { - return output[:limit], nil + output = append(output, items...) + if limit > 0 && len(output) >= limit { + return output[:limit], nil + } } } diff --git a/rbac/objects.go b/rbac/objects.go index dc45de7f8..c02cba0ea 100644 --- a/rbac/objects.go +++ b/rbac/objects.go @@ -29,6 +29,8 @@ var dbResourceObjMap = map[string]string{ "canaries": policy.ObjectCanary, "casbin_rule": policy.ObjectAuth, "catalog_changes": policy.ObjectCatalog, + "change_groups": policy.ObjectCatalog, + "change_groups_summary": policy.ObjectCatalog, "change_types": policy.ObjectDatabasePublic, "changes_by_component": policy.ObjectCatalog, "check_component_relationships": policy.ObjectCanary, diff --git a/schema/config.hcl b/schema/config.hcl index eb51e7280..c53762b9f 100644 --- a/schema/config.hcl +++ b/schema/config.hcl @@ -196,6 +196,12 @@ table "config_changes" { default = sql("now()") } + column "group_id" { + null = true + type = uuid + comment = "optional logical grouping of correlated changes; see change_groups table." + } + primary_key { columns = [column.id] } @@ -205,6 +211,12 @@ table "config_changes" { on_update = NO_ACTION on_delete = CASCADE } + foreign_key "config_changes_group_id_fkey" { + columns = [column.group_id] + ref_columns = [table.change_groups.column.id] + on_update = NO_ACTION + on_delete = SET_NULL + } index "config_changes_created_at_brin_idx" { type = BRIN @@ -230,6 +242,112 @@ table "config_changes" { columns = [column.is_pushed] where = "is_pushed IS FALSE" } + index "config_changes_group_id_idx" { + columns = [column.group_id] + where = "group_id IS NOT NULL" + } + index "config_changes_ungrouped_created_at_idx" { + columns = [column.created_at] + where = "group_id IS NULL" + } +} + +table "change_groups" { + schema = schema.public + comment = "logical grouping of correlated config_changes rows (e.g., pod startup burst, same-commit fan-out, temporary grants)." + + column "id" { + null = false + type = uuid + default = sql("generate_ulid()") + } + column "type" { + null = false + type = text + comment = "typed group kind, e.g. Deployment/v1, TemporaryPermission/v1." + } + column "summary" { + null = false + type = text + default = "" + } + column "correlation_key" { + null = false + type = text + comment = "stable key used to find-or-create a group; unique among open groups of the same type." + } + column "source" { + null = false + type = text + default = "rule" + comment = "explicit, or rule:." + } + column "rule_name" { + null = true + type = text + } + column "status" { + null = false + type = text + default = "open" + } + column "started_at" { + null = false + type = timestamptz + default = sql("now()") + } + column "ended_at" { + null = true + type = timestamptz + } + column "last_member_at" { + null = false + type = timestamptz + default = sql("now()") + } + column "member_count" { + null = false + type = int + default = 0 + } + column "details" { + null = true + type = jsonb + } + column "created_at" { + null = false + type = timestamptz + default = sql("now()") + } + column "updated_at" { + null = false + type = timestamptz + default = sql("now()") + } + + primary_key { + columns = [column.id] + } + + index "change_groups_open_type_correlation_key" { + unique = true + columns = [column.type, column.correlation_key] + where = "status = 'open'" + } + index "change_groups_type_started_at_idx" { + columns = [column.type, column.started_at] + } + index "change_groups_last_member_at_idx" { + columns = [column.last_member_at] + } + index "change_groups_created_at_brin_idx" { + type = BRIN + columns = [column.created_at] + } + index "change_groups_details_gin_idx" { + columns = [column.details] + type = GIN + } } table "config_items" { diff --git a/schema/openapi/change-types.schema.json b/schema/openapi/change-types.schema.json new file mode 100644 index 000000000..7964dbc41 --- /dev/null +++ b/schema/openapi/change-types.schema.json @@ -0,0 +1,1578 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/flanksource/duty/types/config-change-details-schema", + "$ref": "#/$defs/ConfigChangeDetailsSchema", + "$defs": { + "ConfigChangeDetailsSchema": { + "description": "Kind-discriminated schema for config change payloads supported by UnmarshalChangeDetails.", + "oneOf": [ + { "$ref": "#/$defs/UserChangeDetails" }, + { "$ref": "#/$defs/ScreenshotDetails" }, + { "$ref": "#/$defs/PermissionChangeDetails" }, + { "$ref": "#/$defs/GroupMembership" }, + { "$ref": "#/$defs/Identity" }, + { "$ref": "#/$defs/Approval" }, + { "$ref": "#/$defs/GitSource" }, + { "$ref": "#/$defs/HelmSource" }, + { "$ref": "#/$defs/ImageSource" }, + { "$ref": "#/$defs/DatabaseSource" }, + { "$ref": "#/$defs/Source" }, + { "$ref": "#/$defs/Environment" }, + { "$ref": "#/$defs/Event" }, + { "$ref": "#/$defs/Test" }, + { "$ref": "#/$defs/Promotion" }, + { "$ref": "#/$defs/PipelineRun" }, + { "$ref": "#/$defs/Change" }, + { "$ref": "#/$defs/ConfigChange" }, + { "$ref": "#/$defs/Restore" }, + { "$ref": "#/$defs/Backup" }, + { "$ref": "#/$defs/Dimension" }, + { "$ref": "#/$defs/Scale" } + ], + "examples": [ + { + "kind": "UserChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "user_email": "alice@example.com", + "user_type": "human", + "group_id": "group-platform", + "group_name": "platform-admins", + "tenant": "acme" + }, + { + "kind": "PermissionChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "role_id": "role-admin", + "role_name": "cluster-admin", + "role_type": "kubernetes", + "scope": "prod-cluster" + }, + { + "kind": "Promotion/v1", + "id": "evt-promo-1", + "timestamp": "2026-04-10T12:00:00Z", + "from": { + "kind": "Environment/v1", + "name": "staging", + "stage": "Staging" + }, + "to": { + "kind": "Environment/v1", + "name": "production", + "stage": "Production" + }, + "source": { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456" + }, + "path": "deploy/production" + }, + "version": "v1.2.3", + "approvals": [ + { + "kind": "Approval/v1", + "id": "approval-1", + "submitted_by": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "approver": { + "kind": "Identity/v1", + "id": "user-456", + "type": "User", + "name": "bob@example.com" + }, + "stage": "Manual", + "status": "Approved" + } + ], + "artifact": "api" + }, + { + "kind": "Backup/v1", + "id": "backup-42", + "timestamp": "2026-04-10T11:30:00Z", + "end": "2026-04-10T11:36:12Z", + "backup_type": "Snapshot", + "created_by": { + "kind": "Identity/v1", + "type": "System:Auto", + "name": "nightly-backup" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production" + }, + "status": "Completed", + "size": "4.2GB", + "delta": "275MB" + }, + { + "kind": "ConfigChange/v1", + "id": "evt-config-1", + "timestamp": "2026-04-10T12:05:00Z", + "author": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production", + "identifier": "prod-cluster-1" + }, + "source": { + "kind": "Source/v1", + "image": { + "kind": "ImageSource/v1", + "registry": "ghcr.io", + "image": "flanksource/duty", + "version": "v1.2.3", + "sha": "sha256:1234abcd" + } + }, + "changes": [ + { + "kind": "Change/v1", + "path": ".spec.template.spec.containers[0].image", + "from": { + "image": "ghcr.io/flanksource/duty:v1.2.2" + }, + "to": { + "image": "ghcr.io/flanksource/duty:v1.2.3" + }, + "type": "update" + }, + { + "kind": "Change/v1", + "path": ".spec.replicas", + "from": { + "desired": "2" + }, + "to": { + "desired": "3" + }, + "type": "update" + } + ] + } + ] + }, + "StringMap": { + "type": "object", + "description": "String key-value metadata map.", + "additionalProperties": { + "type": "string" + } + }, + "AnyMap": { + "type": "object", + "description": "Object with arbitrary JSON values.", + "additionalProperties": true + }, + "IdentityType": { + "type": "string", + "enum": [ + "User", + "Group", + "Role", + "System:CI", + "System:Auto", + "System:Scan", + "System:Test", + "System:Canary" + ] + }, + "ApprovalStage": { + "type": "string", + "enum": [ + "PreDeployment", + "PostDeployment", + "PrePromotion", + "PostPromotion", + "Manual", + "Automated" + ] + }, + "ApprovalStatus": { + "type": "string", + "enum": [ + "Approved", + "Rejected", + "Pending", + "Expired" + ] + }, + "EnvironmentType": { + "type": "string", + "enum": [ + "Kubernetes", + "Cloud", + "On-Premises", + "Other" + ] + }, + "EnvironmentStage": { + "type": "string", + "enum": [ + "Development", + "Staging", + "Production", + "UAT", + "QA" + ] + }, + "TestingType": { + "type": "string", + "enum": [ + "Unit", + "Integration", + "End-to-End", + "Performance", + "Security" + ] + }, + "TestingStatus": { + "type": "string", + "enum": [ + "Pending", + "Running", + "Passed", + "Failed", + "Skipped", + "Error" + ] + }, + "TestingResult": { + "type": "string", + "enum": [ + "Flaky", + "Failed", + "Passed" + ] + }, + "Status": { + "type": "string", + "enum": [ + "Pending", + "Running", + "Timeout", + "Completed", + "Failed", + "Approved", + "Rejected" + ] + }, + "BackupType": { + "type": "string", + "enum": [ + "Dump", + "Snapshot", + "StorageBackup", + "Offsite", + "OffAccount", + "OffRegion" + ] + }, + "ScalingDimension": { + "type": "string", + "enum": [ + "CPU", + "Memory", + "Replicas", + "Custom" + ] + }, + "Identity": { + "type": "object", + "description": "Versioned identity payload for people, groups, roles, or system actors.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Identity/v1" + }, + "id": { + "type": "string" + }, + "type": { + "$ref": "#/$defs/IdentityType" + }, + "name": { + "type": "string", + "description": "Optional human-readable name for the identity, e.g. user name or group name." + }, + "comment": { + "type": "string", + "description": "Optional comment about the identity, e.g. reason for approval or details about the change." + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + { + "kind": "Identity/v1", + "id": "system-ci", + "type": "System:CI", + "name": "github-actions", + "comment": "Triggered by merge to main" + } + ] + }, + "GitSource": { + "type": "object", + "description": "Git-backed source metadata.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "GitSource/v1" + }, + "url": { + "type": "string", + "format": "uri" + }, + "branch": { + "type": "string" + }, + "commit_sha": { + "type": "string" + }, + "version": { + "type": "string" + }, + "tags": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456", + "version": "v1.2.3", + "tags": "release,v1.2.3" + }, + { + "kind": "GitSource/v1", + "url": "ssh://git@github.com/flanksource/duty.git", + "branch": "release-1.2", + "commit_sha": "fedcba654321" + } + ] + }, + "HelmSource": { + "type": "object", + "description": "Helm chart source metadata.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "HelmSource/v1" + }, + "chart_name": { + "type": "string" + }, + "chart_version": { + "type": "string" + }, + "repo_url": { + "type": "string", + "format": "uri" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "HelmSource/v1", + "chart_name": "ingress-nginx", + "chart_version": "4.10.0", + "repo_url": "https://kubernetes.github.io/ingress-nginx" + }, + { + "kind": "HelmSource/v1", + "chart_name": "duty", + "chart_version": "1.2.3", + "repo_url": "https://charts.example.com/platform" + } + ] + }, + "ImageSource": { + "type": "object", + "description": "Container image source metadata.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "ImageSource/v1" + }, + "registry": { + "type": "string" + }, + "image": { + "type": "string" + }, + "version": { + "type": "string" + }, + "sha": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "ImageSource/v1", + "registry": "ghcr.io", + "image": "flanksource/duty", + "version": "v1.2.3", + "sha": "sha256:1234abcd" + }, + { + "kind": "ImageSource/v1", + "registry": "123456789012.dkr.ecr.us-east-1.amazonaws.com", + "image": "payments/api", + "version": "2026.04.10-1" + } + ] + }, + "DatabaseSource": { + "type": "object", + "description": "Database source metadata.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "DatabaseSource/v1" + }, + "type": { + "type": "string", + "description": "Database type, e.g. PostgreSQL, MySQL, MongoDB." + }, + "name": { + "type": "string", + "description": "Database name." + }, + "schema": { + "type": "string", + "description": "Schema name, e.g. public." + }, + "version": { + "type": "string", + "description": "Database version." + }, + "endpoint": { + "type": "string", + "description": "Server or cluster endpoint." + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "DatabaseSource/v1", + "type": "PostgreSQL", + "name": "appdb", + "schema": "public", + "version": "16", + "endpoint": "appdb.cluster-123.us-east-1.rds.amazonaws.com:5432" + }, + { + "kind": "DatabaseSource/v1", + "type": "MongoDB", + "name": "audit", + "version": "7.0", + "endpoint": "mongo.example.internal:27017" + } + ] + }, + "Source": { + "type": "object", + "description": "Versioned source envelope for Git, Helm, image, or database change provenance.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Source/v1" + }, + "git": { + "$ref": "#/$defs/GitSource" + }, + "helm": { + "$ref": "#/$defs/HelmSource" + }, + "image": { + "$ref": "#/$defs/ImageSource" + }, + "database": { + "$ref": "#/$defs/DatabaseSource" + }, + "kustomization": { + "$ref": "#/$defs/GitSource" + }, + "argocd": { + "$ref": "#/$defs/GitSource" + }, + "other": { + "type": "string" + }, + "path": { + "type": "string", + "description": "Optional path within the source, e.g. file path in git repo or chart path in a Helm repo." + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main" + }, + "path": "deploy/production" + }, + { + "kind": "Source/v1", + "helm": { + "kind": "HelmSource/v1", + "chart_name": "duty", + "chart_version": "1.2.3", + "repo_url": "https://charts.example.com/platform" + }, + "path": "charts/duty" + }, + { + "kind": "Source/v1", + "argocd": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/platform-config.git", + "branch": "main", + "commit_sha": "bbccddeeff00" + }, + "path": "clusters/prod/api" + } + ] + }, + "Environment": { + "type": "object", + "description": "Versioned environment descriptor for change events.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Environment/v1" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "type": { + "$ref": "#/$defs/EnvironmentType", + "description": "Optional type of environment, e.g. Kubernetes, Cloud, On-Premises." + }, + "stage": { + "$ref": "#/$defs/EnvironmentStage", + "description": "Optional stage or lifecycle phase of the environment." + }, + "identifier": { + "type": "string" + }, + "tags": { + "$ref": "#/$defs/StringMap", + "description": "Optional tags for team, owner, namespace, cost center, or other metadata." + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production", + "identifier": "prod-cluster-1", + "tags": { + "cluster": "prod-cluster-1", + "namespace": "platform", + "owner": "platform-team" + } + }, + { + "kind": "Environment/v1", + "name": "staging", + "description": "Pre-production validation environment", + "type": "Cloud", + "stage": "Staging", + "identifier": "staging-account" + } + ] + }, + "Event": { + "type": "object", + "description": "Common event metadata shared by versioned change payloads.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Event/v1" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Event/v1", + "id": "evt-123", + "url": "https://github.com/flanksource/duty/actions/runs/123456789", + "tags": { + "service": "api", + "team": "platform" + }, + "properties": { + "cluster": "prod-cluster-1", + "namespace": "platform" + }, + "timestamp": "2026-04-10T12:00:00Z" + } + ] + }, + "Approval": { + "type": "object", + "description": "Approval event with optional submitter and approver identities.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Approval/v1" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + }, + "submitted_by": { + "$ref": "#/$defs/Identity", + "description": "Optional identity of the person or system that submitted the approval request." + }, + "approver": { + "$ref": "#/$defs/Identity", + "description": "Optional identity of the person or system that approved or rejected the change." + }, + "stage": { + "$ref": "#/$defs/ApprovalStage" + }, + "status": { + "$ref": "#/$defs/ApprovalStatus" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Approval/v1", + "id": "evt-approval-1", + "timestamp": "2026-04-10T11:58:00Z", + "submitted_by": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "approver": { + "kind": "Identity/v1", + "id": "user-456", + "type": "User", + "name": "bob@example.com" + }, + "stage": "Manual", + "status": "Approved" + }, + { + "kind": "Approval/v1", + "id": "evt-approval-2", + "submitted_by": { + "kind": "Identity/v1", + "type": "System:CI", + "name": "github-actions" + }, + "stage": "Automated", + "status": "Pending" + } + ] + }, + "Test": { + "type": "object", + "description": "Versioned test execution event.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Test/v1" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "type": { + "$ref": "#/$defs/TestingType" + }, + "status": { + "$ref": "#/$defs/TestingStatus" + }, + "result": { + "$ref": "#/$defs/TestingResult" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Test/v1", + "id": "evt-test-1", + "timestamp": "2026-04-10T11:50:00Z", + "name": "smoke", + "description": "Validates the production-ready deployment path", + "type": "End-to-End", + "status": "Passed", + "result": "Passed" + }, + { + "kind": "Test/v1", + "id": "evt-test-2", + "name": "container-scan", + "type": "Security", + "status": "Failed", + "result": "Failed" + } + ] + }, + "Promotion": { + "type": "object", + "description": "Promotion event between environments.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Promotion/v1" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + }, + "from": { + "$ref": "#/$defs/Environment", + "description": "Optional source environment for the promotion." + }, + "to": { + "$ref": "#/$defs/Environment", + "description": "Optional target environment for the promotion." + }, + "source": { + "$ref": "#/$defs/Source", + "description": "Optional source for the promotion, e.g. Git repo, Helm chart, image, or database metadata." + }, + "version": { + "type": "string", + "description": "Optional version or identifier for the promoted artifact." + }, + "approvals": { + "type": "array", + "items": { + "$ref": "#/$defs/Approval" + }, + "description": "Optional list of approvals attached to the promotion." + }, + "artifact": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Promotion/v1", + "id": "evt-promo-1", + "timestamp": "2026-04-10T12:00:00Z", + "from": { + "kind": "Environment/v1", + "name": "staging", + "stage": "Staging" + }, + "to": { + "kind": "Environment/v1", + "name": "production", + "stage": "Production" + }, + "source": { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456" + }, + "path": "deploy/production" + }, + "version": "v1.2.3", + "approvals": [ + { + "kind": "Approval/v1", + "id": "approval-1", + "submitted_by": { + "kind": "Identity/v1", + "type": "User", + "name": "alice@example.com" + }, + "approver": { + "kind": "Identity/v1", + "type": "User", + "name": "bob@example.com" + }, + "stage": "Manual", + "status": "Approved" + } + ], + "artifact": "api" + } + ] + }, + "PipelineRun": { + "type": "object", + "description": "Pipeline run event scoped to an environment.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "PipelineRun/v1" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + }, + "environment": { + "$ref": "#/$defs/Environment" + }, + "status": { + "$ref": "#/$defs/Status" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "PipelineRun/v1", + "id": "evt-pipeline-1", + "url": "https://github.com/flanksource/duty/actions/runs/123456789", + "timestamp": "2026-04-10T11:45:00Z", + "environment": { + "kind": "Environment/v1", + "name": "production", + "stage": "Production" + }, + "status": "Running" + }, + { + "kind": "PipelineRun/v1", + "id": "evt-pipeline-2", + "environment": { + "kind": "Environment/v1", + "name": "staging", + "stage": "Staging" + }, + "status": "Completed" + } + ] + }, + "Change": { + "type": "object", + "description": "Single field-level change within a larger config change payload.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Change/v1" + }, + "path": { + "type": "string" + }, + "from": { + "$ref": "#/$defs/AnyMap" + }, + "to": { + "$ref": "#/$defs/AnyMap" + }, + "type": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Change/v1", + "path": ".spec.replicas", + "from": { + "desired": "2" + }, + "to": { + "desired": "3" + }, + "type": "update" + }, + { + "kind": "Change/v1", + "path": ".metadata.annotations.release", + "to": { + "release": "2026.04.10" + }, + "type": "create" + } + ] + }, + "ConfigChange": { + "type": "object", + "description": "Top-level config change payload with author, environment, source, and individual changes.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "ConfigChange/v1" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + }, + "author": { + "$ref": "#/$defs/Identity" + }, + "changes": { + "type": "array", + "items": { + "$ref": "#/$defs/Change" + } + }, + "environment": { + "$ref": "#/$defs/Environment" + }, + "source": { + "$ref": "#/$defs/Source" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "ConfigChange/v1", + "id": "evt-config-1", + "url": "https://github.com/flanksource/duty/pull/123", + "timestamp": "2026-04-10T12:05:00Z", + "author": { + "kind": "Identity/v1", + "id": "user-123", + "type": "User", + "name": "alice@example.com" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production", + "identifier": "prod-cluster-1" + }, + "source": { + "kind": "Source/v1", + "git": { + "kind": "GitSource/v1", + "url": "https://github.com/flanksource/duty.git", + "branch": "main", + "commit_sha": "abc123def456" + }, + "path": "deploy/production" + }, + "changes": [ + { + "kind": "Change/v1", + "path": ".spec.template.spec.containers[0].image", + "from": { + "image": "ghcr.io/flanksource/duty:v1.2.2" + }, + "to": { + "image": "ghcr.io/flanksource/duty:v1.2.3" + }, + "type": "update" + }, + { + "kind": "Change/v1", + "path": ".spec.replicas", + "from": { + "desired": "2" + }, + "to": { + "desired": "3" + }, + "type": "update" + } + ] + } + ] + }, + "Restore": { + "type": "object", + "description": "Restore event between environments.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Restore/v1" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + }, + "from": { + "$ref": "#/$defs/Environment", + "description": "Optional source environment for the restore." + }, + "to": { + "$ref": "#/$defs/Environment", + "description": "Optional target environment for the restore." + }, + "source": { + "$ref": "#/$defs/Source", + "description": "Optional source metadata for the restore." + }, + "status": { + "$ref": "#/$defs/Status", + "description": "Optional status of the restore." + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Restore/v1", + "id": "evt-restore-1", + "timestamp": "2026-04-10T13:10:00Z", + "from": { + "kind": "Environment/v1", + "name": "backup-storage", + "type": "Cloud", + "stage": "Production" + }, + "to": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production" + }, + "source": { + "kind": "Source/v1", + "database": { + "kind": "DatabaseSource/v1", + "type": "PostgreSQL", + "name": "appdb", + "schema": "public", + "version": "16" + } + }, + "status": "Completed" + } + ] + }, + "Backup": { + "type": "object", + "description": "Backup event with actor, environment, event metadata, and backup result fields.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Backup/v1" + }, + "backup_type": { + "$ref": "#/$defs/BackupType" + }, + "created_by": { + "$ref": "#/$defs/Identity" + }, + "environment": { + "$ref": "#/$defs/Environment" + }, + "id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "tags": { + "$ref": "#/$defs/StringMap" + }, + "properties": { + "$ref": "#/$defs/StringMap" + }, + "timestamp": { + "type": "string" + }, + "end": { + "type": "string" + }, + "status": { + "$ref": "#/$defs/Status" + }, + "size": { + "type": "string" + }, + "delta": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Backup/v1", + "id": "backup-42", + "backup_type": "Snapshot", + "created_by": { + "kind": "Identity/v1", + "type": "System:Auto", + "name": "nightly-backup" + }, + "environment": { + "kind": "Environment/v1", + "name": "production", + "type": "Kubernetes", + "stage": "Production" + }, + "timestamp": "2026-04-10T11:30:00Z", + "end": "2026-04-10T11:36:12Z", + "status": "Completed", + "size": "4.2GB", + "delta": "275MB" + } + ] + }, + "Dimension": { + "type": "object", + "description": "Dimension values used by scale events.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Dimension/v1" + }, + "min": { + "type": "string" + }, + "max": { + "type": "string" + }, + "desired": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Dimension/v1", + "min": "2", + "max": "10", + "desired": "3" + } + ] + }, + "Scale": { + "type": "object", + "description": "Scale event payload with before and after dimension values.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Scale/v1" + }, + "dimension": { + "$ref": "#/$defs/ScalingDimension" + }, + "previous_value": { + "$ref": "#/$defs/Dimension" + }, + "value": { + "$ref": "#/$defs/Dimension" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Scale/v1", + "dimension": "Replicas", + "previous_value": { + "kind": "Dimension/v1", + "min": "2", + "max": "10", + "desired": "2" + }, + "value": { + "kind": "Dimension/v1", + "min": "2", + "max": "10", + "desired": "5" + } + }, + { + "kind": "Scale/v1", + "dimension": "CPU", + "previous_value": { + "kind": "Dimension/v1", + "desired": "500m" + }, + "value": { + "kind": "Dimension/v1", + "desired": "1500m" + } + } + ] + }, + "UserChangeDetails": { + "type": "object", + "description": "User change payload.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "UserChange/v1" + }, + "user_id": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "user_email": { + "type": "string" + }, + "user_type": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "group_name": { + "type": "string" + }, + "tenant": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "UserChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "user_email": "alice@example.com", + "user_type": "human", + "group_id": "group-platform", + "group_name": "platform-admins" + }, + { + "kind": "UserChange/v1", + "user_id": "svc-789", + "user_name": "deploy-bot", + "user_type": "service-account", + "tenant": "acme" + } + ] + }, + "ScreenshotDetails": { + "type": "object", + "description": "Screenshot artifact metadata.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "Screenshot/v1" + }, + "artifact_id": { + "type": "string" + }, + "url": { + "type": "string", + "format": "uri" + }, + "content_type": { + "type": "string" + }, + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "Screenshot/v1", + "artifact_id": "artifact-123", + "url": "https://example.com/screenshot.png", + "content_type": "image/png", + "width": 1920, + "height": 1080 + }, + { + "kind": "Screenshot/v1", + "artifact_id": "artifact-456", + "url": "https://example.com/error-state.jpg", + "content_type": "image/jpeg", + "width": 1280, + "height": 720 + } + ] + }, + "PermissionChangeDetails": { + "type": "object", + "description": "Permission or role assignment change payload.", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "PermissionChange/v1" + }, + "user_id": { + "type": "string" + }, + "user_name": { + "type": "string" + }, + "group_id": { + "type": "string" + }, + "group_name": { + "type": "string" + }, + "role_id": { + "type": "string" + }, + "role_name": { + "type": "string" + }, + "role_type": { + "type": "string" + }, + "scope": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "PermissionChange/v1", + "user_id": "user-123", + "user_name": "alice@example.com", + "group_id": "group-platform", + "group_name": "platform-admins", + "role_id": "role-admin", + "role_name": "admin", + "role_type": "kubernetes", + "scope": "cluster-1" + }, + { + "kind": "PermissionChange/v1", + "group_id": "group-observability", + "group_name": "observability-team", + "role_id": "role-viewer", + "role_name": "viewer", + "role_type": "rbac", + "scope": "grafana-prod" + } + ] + }, + "GroupMembership": { + "type": "object", + "description": "Group membership change payload (member added to or removed from a group).", + "required": [ + "kind" + ], + "properties": { + "kind": { + "const": "GroupMembership/v1" + }, + "group": { + "$ref": "#/$defs/Identity" + }, + "member": { + "$ref": "#/$defs/Identity" + }, + "action": { + "type": "string", + "enum": ["Added", "Removed"] + }, + "tenant": { + "type": "string" + } + }, + "additionalProperties": false, + "examples": [ + { + "kind": "GroupMembership/v1", + "group": { + "kind": "Identity/v1", + "id": "group-platform", + "name": "platform-admins", + "type": "Group" + }, + "member": { + "kind": "Identity/v1", + "id": "user-123", + "name": "alice@example.com", + "type": "User" + }, + "action": "Added", + "tenant": "acme" + }, + { + "kind": "GroupMembership/v1", + "group": { + "kind": "Identity/v1", + "id": "group-observability", + "name": "observability-team", + "type": "Group" + }, + "member": { + "kind": "Identity/v1", + "id": "svc-789", + "name": "deploy-bot", + "type": "User" + }, + "action": "Removed" + } + ] + } + } +} diff --git a/tests/change_groups_test.go b/tests/change_groups_test.go new file mode 100644 index 000000000..8995521c1 --- /dev/null +++ b/tests/change_groups_test.go @@ -0,0 +1,435 @@ +package tests + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + ginkgo "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + + "github.com/flanksource/duty/changegroup" + "github.com/flanksource/duty/context" + "github.com/flanksource/duty/models" + "github.com/flanksource/duty/query" + "github.com/flanksource/duty/tests/fixtures/dummy" + "github.com/flanksource/duty/types" +) + +// stubProgram wraps a Go closure so a test can build an Evaluator without CEL. +// The Program value stored on GroupingRule is one of these closures. +type stubProgram struct { + boolFn func(changegroup.Env) (bool, error) + stringFn func(changegroup.Env) (string, error) + detailsFn func(changegroup.Env) (types.GroupType, error) +} + +// stubEvaluator implements changegroup.Evaluator by treating each expression +// string as an opaque key into a map that the test fills out before Validate. +type stubEvaluator struct { + programs map[string]*stubProgram +} + +func newStubEvaluator() *stubEvaluator { return &stubEvaluator{programs: map[string]*stubProgram{}} } + +func (s *stubEvaluator) register(expr string, p *stubProgram) { s.programs[expr] = p } + +func (s *stubEvaluator) Compile(expr string, kind changegroup.ExprKind) (changegroup.Program, error) { + p, ok := s.programs[expr] + if !ok { + return nil, ginkgoError("stub evaluator: no program registered for expression %q", expr) + } + _ = kind + return p, nil +} + +func (s *stubEvaluator) EvalBool(p changegroup.Program, env changegroup.Env) (bool, error) { + return p.(*stubProgram).boolFn(env) +} +func (s *stubEvaluator) EvalString(p changegroup.Program, env changegroup.Env) (string, error) { + return p.(*stubProgram).stringFn(env) +} +func (s *stubEvaluator) EvalGroupDetails(p changegroup.Program, env changegroup.Env) (types.GroupType, error) { + return p.(*stubProgram).detailsFn(env) +} + +func ginkgoError(format string, args ...any) error { + return &stubError{msg: sprintf(format, args...)} +} + +type stubError struct{ msg string } + +func (e *stubError) Error() string { return e.msg } + +func sprintf(format string, args ...any) string { + // Tiny fmt shim so this file doesn't import "fmt" just for one call. + b := make([]byte, 0, len(format)+64) + ai := 0 + for i := 0; i < len(format); i++ { + if format[i] == '%' && i+1 < len(format) && format[i+1] == 'q' && ai < len(args) { + b = append(b, '"') + b = append(b, []byte(args[ai].(string))...) + b = append(b, '"') + i++ + ai++ + continue + } + b = append(b, format[i]) + } + return string(b) +} + +// insertChange persists a config_change row and returns it with ID populated. +func insertChange(ctx context.Context, c models.ConfigChange) models.ConfigChange { + if c.ID == "" { + c.ID = uuid.New().String() + } + if c.CreatedAt == nil { + c.CreatedAt = lo.ToPtr(time.Now().UTC()) + } + Expect(ctx.DB().Create(&c).Error).To(Succeed()) + return c +} + +var _ = ginkgo.Describe("changegroup engine", ginkgo.Ordered, func() { + var ( + ev *stubEvaluator + engine *changegroup.Engine + ) + + ginkgo.BeforeEach(func() { + ev = newStubEvaluator() + engine = nil + }) + + ginkgo.AfterEach(func() { + // Clean rows we inserted so each spec is independent. + Expect(DefaultContext.DB().Exec(`DELETE FROM config_changes WHERE source = 'test-changegroup'`).Error).To(Succeed()) + Expect(DefaultContext.DB().Exec(`DELETE FROM change_groups WHERE source LIKE 'rule:test-%' OR source = 'explicit'`).Error).To(Succeed()) + }) + + ginkgo.It("pod startup: time-bucket groups many changes on one config into one group", func() { + // Rule: same_config + 5s bucket + @changed. + ev.register("startup_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + cfgID := env.Change["config_id"].(string) + return cfgID + ":bucket0", nil + }, + }) + ev.register("startup_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + cfg, _ := uuid.Parse(env.Change["config_id"].(string)) + return types.StartupGroup{ + ConfigID: cfg, + Reason: "test", + RestartCount: len(env.Changes), + }, nil + }, + }) + ev.register("startup_summary", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + return "pod startup burst", nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{ + { + Name: "test-pod-startup", + Scope: changegroup.Scope{Kind: changegroup.ScopeSameConfig}, + Window: changegroup.Duration(5 * time.Second), + ChangeTypes: []string{"UPDATE", "diff"}, + Key: "startup_key", + Details: "startup_details", + Summary: "startup_summary", + }, + }) + Expect(err).ToNot(HaveOccurred()) + + cfg := dummy.NginxIngressPod.ID.String() + for i := 0; i < 5; i++ { + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: cfg, + ChangeType: "diff", + Source: "test-changegroup", + Summary: "startup event", + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + } + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeStartup, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + Expect(groups[0].Summary).To(Equal("pod startup burst")) + Expect(groups[0].MemberCount).To(Equal(5)) + + members, err := query.GetGroupMembers(DefaultContext, groups[0].ID) + Expect(err).ToNot(HaveOccurred()) + Expect(members).To(HaveLen(5)) + }) + + ginkgo.It("deployment fan-out: same image across different configs shares one group", func() { + ev.register("deploy_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + d, _ := env.Change["details"].(types.JSON) + var m map[string]any + if len(d) > 0 { + _ = json.Unmarshal(d, &m) + } + img, _ := m["new_image"].(string) + return img, nil + }, + }) + ev.register("deploy_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + // Aggregate config_ids from all members. + ids := make([]uuid.UUID, 0, len(env.Changes)) + for _, m := range env.Changes { + if id, err := uuid.Parse(m["config_id"].(string)); err == nil { + ids = append(ids, id) + } + } + d, _ := env.Change["details"].(types.JSON) + var details map[string]any + if len(d) > 0 { + _ = json.Unmarshal(d, &details) + } + img, _ := details["new_image"].(string) + return types.DeploymentGroup{ + Image: img, + Version: "v1.2.3", + TargetConfigIDs: ids, + }, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{ + { + Name: "test-deployment-fanout", + Scope: changegroup.Scope{Kind: changegroup.ScopeAll}, + Window: changegroup.Duration(2 * time.Minute), + ChangeTypes: []string{types.ChangeTypeDeployment}, + Key: "deploy_key", + Details: "deploy_details", + }, + }) + Expect(err).ToNot(HaveOccurred()) + + configs := []models.ConfigItem{ + dummy.EC2InstanceA, + dummy.EC2InstanceB, + dummy.NginxIngressPod, + } + for _, ci := range configs { + details := types.JSON(`{"new_image":"registry/app:v1.2.3"}`) + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: ci.ID.String(), + ChangeType: types.ChangeTypeDeployment, + Source: "test-changegroup", + Summary: "deployed " + ci.ID.String(), + Details: details, + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + } + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeDeployment, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + Expect(groups[0].MemberCount).To(Equal(len(configs))) + + typed, err := groups[0].TypedDetails() + Expect(err).ToNot(HaveOccurred()) + dep := typed.(types.DeploymentGroup) + Expect(dep.Image).To(Equal("registry/app:v1.2.3")) + Expect(dep.TargetConfigIDs).To(HaveLen(len(configs))) + }) + + ginkgo.It("temporary permission: grant then revoke close the group with duration", func() { + ev.register("tp_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { + d, _ := env.Change["details"].(types.JSON) + var m map[string]any + _ = json.Unmarshal(d, &m) + return m["user_id"].(string) + "|" + m["role_id"].(string), nil + }, + }) + ev.register("tp_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + var grant, revoke *uuid.UUID + for _, m := range env.Changes { + id, _ := uuid.Parse(m["id"].(string)) + switch m["change_type"].(string) { + case types.ChangeTypePermissionAdded: + tmp := id + if grant == nil { + grant = &tmp + } + case types.ChangeTypePermissionRemoved: + tmp := id + if revoke == nil { + revoke = &tmp + } + } + } + d, _ := env.Change["details"].(types.JSON) + var dm map[string]any + _ = json.Unmarshal(d, &dm) + return types.TemporaryPermissionGroup{ + UserID: dm["user_id"].(string), + RoleID: dm["role_id"].(string), + Scope: dm["scope"].(string), + GrantChangeID: grant, + RevokeChangeID: revoke, + }, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{ + { + Name: "test-temporary-permission", + Scope: changegroup.Scope{Kind: changegroup.ScopeByDetailsField, Field: "user_id"}, + Window: changegroup.Duration(720 * time.Hour), + CloseAfter: 0, + ChangeTypes: []string{types.ChangeTypePermissionAdded, types.ChangeTypePermissionRemoved}, + Key: "tp_key", + Details: "tp_details", + }, + }) + Expect(err).ToNot(HaveOccurred()) + + cfg := dummy.EKSCluster.ID.String() + t0 := time.Now().UTC().Add(-time.Hour) + grant := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: cfg, + ChangeType: types.ChangeTypePermissionAdded, + Source: "test-changegroup", + Details: types.JSON(`{"user_id":"alice","role_id":"admin","scope":"cluster-1"}`), + CreatedAt: &t0, + }) + Expect(engine.Evaluate(DefaultContext, &grant)).To(Succeed()) + + t1 := t0.Add(time.Hour) + revoke := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: cfg, + ChangeType: types.ChangeTypePermissionRemoved, + Source: "test-changegroup", + Details: types.JSON(`{"user_id":"alice","role_id":"admin","scope":"cluster-1"}`), + CreatedAt: &t1, + }) + Expect(engine.Evaluate(DefaultContext, &revoke)).To(Succeed()) + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeTemporaryPermission, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + Expect(groups[0].MemberCount).To(Equal(2)) + + typed, err := groups[0].TypedDetails() + Expect(err).ToNot(HaveOccurred()) + tp := typed.(types.TemporaryPermissionGroup) + Expect(tp.GrantChangeID).ToNot(BeNil()) + Expect(tp.RevokeChangeID).ToNot(BeNil()) + Expect(*tp.GrantChangeID).To(Equal(uuid.MustParse(grant.ID))) + Expect(*tp.RevokeChangeID).To(Equal(uuid.MustParse(revoke.ID))) + }) + + ginkgo.It("explicit path: engine leaves producer-assigned group_id alone", func() { + // Rule that would otherwise match — but change already has GroupID. + ev.register("noop_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { return "noop", nil }, + }) + ev.register("noop_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + return types.CustomGroup{Fields: map[string]any{"k": "v"}}, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{{ + Name: "test-should-not-run", + Scope: changegroup.Scope{Kind: changegroup.ScopeAll}, + Window: changegroup.Duration(time.Minute), + Key: "noop_key", + Details: "noop_details", + }}) + Expect(err).ToNot(HaveOccurred()) + + gid, err := changegroup.CreateTyped(DefaultContext, + types.CustomGroup{Fields: map[string]any{"origin": "playbook"}}, + "playbook-driven", + ) + Expect(err).ToNot(HaveOccurred()) + + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: dummy.EC2InstanceA.ID.String(), + ChangeType: "Pulled", + Source: "test-changegroup", + GroupID: &gid, + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + + g, err := query.GetChangeGroup(DefaultContext, gid) + Expect(err).ToNot(HaveOccurred()) + Expect(g.Source).To(Equal(models.ChangeGroupSourceExplicit)) + Expect(g.MemberCount).To(Equal(1), "explicit assign should have been counted by the 047 trigger") + }) + + ginkgo.It("re-evaluation: each attach sees growing changes list", func() { + // Record how many members each evaluation saw. + var observed []int + ev.register("re_key", &stubProgram{ + stringFn: func(env changegroup.Env) (string, error) { return "single-key", nil }, + }) + ev.register("re_details", &stubProgram{ + detailsFn: func(env changegroup.Env) (types.GroupType, error) { + observed = append(observed, len(env.Changes)) + ids := make([]uuid.UUID, 0, len(env.Changes)) + for _, m := range env.Changes { + if id, err := uuid.Parse(m["config_id"].(string)); err == nil { + ids = append(ids, id) + } + } + return types.DeploymentGroup{Image: "x", TargetConfigIDs: ids}, nil + }, + }) + + var err error + engine, err = changegroup.New(ev, []changegroup.GroupingRule{{ + Name: "test-re-eval", + Scope: changegroup.Scope{Kind: changegroup.ScopeAll}, + Window: changegroup.Duration(time.Minute), + ChangeTypes: []string{types.ChangeTypeDeployment}, + Key: "re_key", + Details: "re_details", + }}) + Expect(err).ToNot(HaveOccurred()) + + for _, ci := range []models.ConfigItem{dummy.EC2InstanceA, dummy.EC2InstanceB, dummy.NginxIngressPod} { + c := insertChange(DefaultContext, models.ConfigChange{ + ConfigID: ci.ID.String(), + ChangeType: types.ChangeTypeDeployment, + Source: "test-changegroup", + }) + Expect(engine.Evaluate(DefaultContext, &c)).To(Succeed()) + } + + Expect(observed).To(Equal([]int{1, 2, 3})) + + groups, err := query.FindChangeGroups(DefaultContext, query.ChangeGroupsSearchRequest{ + Type: types.GroupTypeDeployment, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(groups).To(HaveLen(1)) + typed, err := groups[0].TypedDetails() + Expect(err).ToNot(HaveOccurred()) + Expect(typed.(types.DeploymentGroup).TargetConfigIDs).To(HaveLen(3)) + }) +}) diff --git a/tests/fixtures/dummy/all.go b/tests/fixtures/dummy/all.go index b2ffd0b6e..cf90a5fdb 100644 --- a/tests/fixtures/dummy/all.go +++ b/tests/fixtures/dummy/all.go @@ -1021,26 +1021,26 @@ func GenerateDynamicDummyData(db *gorm.DB) DummyData { var EKSClusterCreateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, CreatedAt: &DummyYearOldDate, } var EKSClusterUpdateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, } var EKSClusterDeleteChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "DELETE", + ChangeType: types.ChangeTypeDelete, } var KubernetesNodeAChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: KubernetesNodeA.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, } var configChanges = []models.ConfigChange{ diff --git a/tests/fixtures/dummy/application_data.go b/tests/fixtures/dummy/application_data.go index 5ffeaa0db..12478598b 100644 --- a/tests/fixtures/dummy/application_data.go +++ b/tests/fixtures/dummy/application_data.go @@ -80,7 +80,7 @@ var RDSBackupChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: RDSInstance.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "AWS", Details: types.JSON(`{"status":"success","size":"4.2GB"}`), CreatedAt: &appT48h, @@ -88,7 +88,7 @@ var RDSBackupChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: RDSInstance.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "AWS", Details: types.JSON(`{"status":"success","size":"4.3GB"}`), CreatedAt: &appT24h, @@ -96,7 +96,7 @@ var RDSBackupChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: RDSInstance.ID.String(), - ChangeType: "BackupRestored", + ChangeType: types.ChangeTypeBackupRestored, Source: "AWS", Details: types.JSON(`{"status":"success"}`), CreatedAt: &appT12h, @@ -107,7 +107,7 @@ var DeploymentDiffChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: IncidentCommanderDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "low", Summary: "image updated: v1.2.3 -> v1.2.4", @@ -116,7 +116,7 @@ var DeploymentDiffChanges = []models.ConfigChange{ { ID: uuid.New().String(), ConfigID: IncidentCommanderDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "info", Summary: "replicas scaled: 2 -> 3", @@ -274,7 +274,7 @@ var KubernetesAppDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("a1b2c3d4-e5f6-7890-abcd-000000000010").String(), ConfigID: KubernetesAppDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "low", Summary: "image updated: v2.0.9 -> v2.1.0", @@ -283,7 +283,7 @@ var KubernetesAppDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("a1b2c3d4-e5f6-7890-abcd-000000000011").String(), ConfigID: KubernetesAppDeployment.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "info", Summary: "replicas scaled: 2 -> 3", @@ -292,7 +292,7 @@ var KubernetesAppDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("a1b2c3d4-e5f6-7890-abcd-000000000012").String(), ConfigID: KubernetesAppIngress.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "kubernetes", Severity: "medium", Summary: "TLS certificate renewed", @@ -367,7 +367,7 @@ var MSSQLBackupChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000010").String(), ConfigID: MSSQLProdDatabase.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "mssql", Details: types.JSON(`{"status":"success","size":"12.4GB","type":"FULL"}`), CreatedAt: &appT48h, @@ -375,7 +375,7 @@ var MSSQLBackupChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000011").String(), ConfigID: MSSQLProdDatabase.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "mssql", Details: types.JSON(`{"status":"success","size":"12.7GB","type":"FULL"}`), CreatedAt: &appT24h, @@ -386,7 +386,7 @@ var MSSQLDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000012").String(), ConfigID: MSSQLProdDatabase.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "mssql", Severity: "medium", Summary: "schema migration: added column orders.fulfilled_at", @@ -395,7 +395,7 @@ var MSSQLDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("b2c3d4e5-f6a7-8901-bcde-000000000013").String(), ConfigID: MSSQLServer.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "mssql", Severity: "low", Summary: "server collation updated", @@ -530,7 +530,7 @@ var PopAPIRepoDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("c3d4e5f6-a7b8-9012-cdef-000000000010").String(), ConfigID: PopAPIRepo.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "github", Severity: "info", Summary: "PR #124 merged: add connection pooling", @@ -539,7 +539,7 @@ var PopAPIRepoDiffChanges = []models.ConfigChange{ { ID: uuid.MustParse("c3d4e5f6-a7b8-9012-cdef-000000000011").String(), ConfigID: PopAPIRepo.ID.String(), - ChangeType: "diff", + ChangeType: types.ChangeTypeDiff, Source: "github", Severity: "low", Summary: "tag pushed: v0.9.3", @@ -551,7 +551,7 @@ var PopAPIDBChanges = []models.ConfigChange{ { ID: uuid.MustParse("c3d4e5f6-a7b8-9012-cdef-000000000012").String(), ConfigID: PopAPIDatabase.ID.String(), - ChangeType: "BackupCompleted", + ChangeType: types.ChangeTypeBackupCompleted, Source: "postgresql", Details: types.JSON(`{"status":"success","size":"512MB"}`), CreatedAt: &appT24h, @@ -612,7 +612,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000010").String(), ConfigID: AzDOBuildPipeline.ID.String(), - ChangeType: "PipelineRunStarted", + ChangeType: types.ChangeTypePipelineRunStarted, Source: "azuredevops", Severity: "info", Summary: "build #88 started on main", @@ -621,7 +621,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000011").String(), ConfigID: AzDOBuildPipeline.ID.String(), - ChangeType: "PipelineRunCompleted", + ChangeType: types.ChangeTypePipelineRunCompleted, Source: "azuredevops", Severity: "info", Summary: "build #88 succeeded in 4m32s", @@ -630,7 +630,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000012").String(), ConfigID: AzDOBuildPipeline.ID.String(), - ChangeType: "PipelineRunFailed", + ChangeType: types.ChangeTypePipelineRunFailed, Source: "azuredevops", Severity: "high", Summary: "build #87 failed: test stage timed out", @@ -639,7 +639,7 @@ var AzDOPipelineChanges = []models.ConfigChange{ { ID: uuid.MustParse("d4e5f6a7-b8c9-0123-defa-000000000013").String(), ConfigID: AzDOReleasePipeline.ID.String(), - ChangeType: "PipelineRunCompleted", + ChangeType: types.ChangeTypePipelineRunCompleted, Source: "azuredevops", Severity: "info", Summary: "release #12 deployed to production", diff --git a/tests/fixtures/dummy/config_changes.go b/tests/fixtures/dummy/config_changes.go index d2a2fe0fd..08e3c3016 100644 --- a/tests/fixtures/dummy/config_changes.go +++ b/tests/fixtures/dummy/config_changes.go @@ -7,12 +7,13 @@ import ( "github.com/samber/lo" "github.com/flanksource/duty/models" + "github.com/flanksource/duty/types" ) var EKSClusterCreateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, CreatedAt: &DummyYearOldDate, Severity: models.SeverityMedium, Source: "CloudTrail", @@ -24,7 +25,7 @@ var EKSClusterCreateChange = models.ConfigChange{ var EKSClusterUpdateChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 24)), Severity: models.SeverityLow, Source: "CloudTrail", @@ -36,7 +37,7 @@ var EKSClusterUpdateChange = models.ConfigChange{ var EKSClusterDeleteChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: EKSCluster.ID.String(), - ChangeType: "DELETE", + ChangeType: types.ChangeTypeDelete, CreatedAt: &DummyNow, Severity: models.SeverityHigh, Source: "CloudTrail", @@ -48,7 +49,7 @@ var EKSClusterDeleteChange = models.ConfigChange{ var KubernetesNodeAChange = models.ConfigChange{ ID: uuid.New().String(), ConfigID: KubernetesNodeA.ID.String(), - ChangeType: "CREATE", + ChangeType: types.ChangeTypeCreate, CreatedAt: &DummyYearOldDate, Severity: models.SeverityInfo, Source: "Kubernetes", @@ -61,7 +62,7 @@ var KubernetesNodeAChange = models.ConfigChange{ var NginxHelmReleaseUpgradeV1 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: NginxHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 72)), // 3 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -73,7 +74,7 @@ var NginxHelmReleaseUpgradeV1 = models.ConfigChange{ var NginxHelmReleaseUpgradeV2 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: NginxHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 48)), // 2 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -85,7 +86,7 @@ var NginxHelmReleaseUpgradeV2 = models.ConfigChange{ var NginxHelmReleaseUpgradeV3 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: NginxHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 24)), // 1 day ago Severity: models.SeverityInfo, Source: "Flux", @@ -98,7 +99,7 @@ var NginxHelmReleaseUpgradeV3 = models.ConfigChange{ var RedisHelmReleaseUpgradeV1 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: RedisHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 96)), // 4 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -110,7 +111,7 @@ var RedisHelmReleaseUpgradeV1 = models.ConfigChange{ var RedisHelmReleaseUpgradeV2 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: RedisHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 60)), // 2.5 days ago Severity: models.SeverityInfo, Source: "Flux", @@ -122,7 +123,7 @@ var RedisHelmReleaseUpgradeV2 = models.ConfigChange{ var RedisHelmReleaseUpgradeV3 = models.ConfigChange{ ID: uuid.New().String(), ConfigID: RedisHelmRelease.ID.String(), - ChangeType: "UPDATE", + ChangeType: types.ChangeTypeUpdate, CreatedAt: lo.ToPtr(DummyNow.Add(-time.Hour * 36)), // 1.5 days ago Severity: models.SeverityInfo, Source: "Flux", diff --git a/types/config_change_groups.go b/types/config_change_groups.go new file mode 100644 index 000000000..0bad9528c --- /dev/null +++ b/types/config_change_groups.go @@ -0,0 +1,171 @@ +package types + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" +) + +// GroupType is implemented by all typed group detail structs +// for the ChangeGroup.Details field. +type GroupType interface { + Kind() string +} + +// Change group type constants. +const ( + GroupTypeStartup = "Startup/v1" + GroupTypeDeployment = "Deployment/v1" + GroupTypePromotion = "Promotion/v1" + GroupTypeTemporaryPermission = "TemporaryPermission/v1" + GroupTypeIncidentResponse = "IncidentResponse/v1" + GroupTypeCustom = "Custom/v1" +) + +// ConfigChangeGroupDetailsSchema is a union type used for JSON schema generation. +// It is never instantiated at runtime. +type ConfigChangeGroupDetailsSchema struct { + Startup *StartupGroup `json:"Startup/v1,omitempty"` + Deployment *DeploymentGroup `json:"Deployment/v1,omitempty"` + Promotion *PromotionGroup `json:"Promotion/v1,omitempty"` + TemporaryPermission *TemporaryPermissionGroup `json:"TemporaryPermission/v1,omitempty"` + IncidentResponse *IncidentResponseGroup `json:"IncidentResponse/v1,omitempty"` + Custom *CustomGroup `json:"Custom/v1,omitempty"` +} + +type StartupGroup struct { + ConfigID uuid.UUID `json:"config_id"` + Reason string `json:"reason,omitempty"` + RestartCount int `json:"restart_count,omitempty"` +} + +func (g StartupGroup) Kind() string { return GroupTypeStartup } +func (g StartupGroup) MarshalJSON() ([]byte, error) { + type raw StartupGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type DeploymentGroup struct { + Image string `json:"image,omitempty"` + Version string `json:"version,omitempty"` + Commit string `json:"commit,omitempty"` + Strategy string `json:"strategy,omitempty"` + TargetConfigIDs []uuid.UUID `json:"target_config_ids,omitempty" merge:"append"` +} + +func (g DeploymentGroup) Kind() string { return GroupTypeDeployment } +func (g DeploymentGroup) MarshalJSON() ([]byte, error) { + type raw DeploymentGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type PromotionGroup struct { + FromEnvironment string `json:"from_environment,omitempty"` + ToEnvironment string `json:"to_environment,omitempty"` + Version string `json:"version,omitempty"` + Artifact string `json:"artifact,omitempty"` + PromotionChangeID *uuid.UUID `json:"promotion_change_id,omitempty" merge:"firstSet"` + TargetDeploymentIDs []uuid.UUID `json:"target_deployment_ids,omitempty" merge:"append"` +} + +func (g PromotionGroup) Kind() string { return GroupTypePromotion } +func (g PromotionGroup) MarshalJSON() ([]byte, error) { + type raw PromotionGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type TemporaryPermissionGroup struct { + UserID string `json:"user_id,omitempty"` + RoleID string `json:"role_id,omitempty"` + Scope string `json:"scope,omitempty"` + GrantChangeID *uuid.UUID `json:"grant_change_id,omitempty" merge:"firstSet"` + RevokeChangeID *uuid.UUID `json:"revoke_change_id,omitempty" merge:"firstSet"` + DurationSeconds *int64 `json:"duration_seconds,omitempty"` +} + +func (g TemporaryPermissionGroup) Kind() string { return GroupTypeTemporaryPermission } +func (g TemporaryPermissionGroup) MarshalJSON() ([]byte, error) { + type raw TemporaryPermissionGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type IncidentResponseGroup struct { + IncidentID string `json:"incident_id,omitempty"` + OpenedAt time.Time `json:"opened_at,omitempty" merge:"min"` + ClosedAt time.Time `json:"closed_at,omitempty" merge:"max"` + PlaybookRunIDs []uuid.UUID `json:"playbook_run_ids,omitempty" merge:"append"` +} + +func (g IncidentResponseGroup) Kind() string { return GroupTypeIncidentResponse } +func (g IncidentResponseGroup) MarshalJSON() ([]byte, error) { + type raw IncidentResponseGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +type CustomGroup struct { + Fields map[string]any `json:"fields,omitempty" merge:"mapMerge"` +} + +func (g CustomGroup) Kind() string { return GroupTypeCustom } +func (g CustomGroup) MarshalJSON() ([]byte, error) { + type raw CustomGroup + return marshalWithKind(g.Kind(), raw(g)) +} + +// UnmarshalGroupDetails inspects the "kind" envelope and returns the matching +// concrete GroupType value. +func UnmarshalGroupDetails(raw json.RawMessage) (GroupType, error) { + if len(raw) == 0 || string(raw) == "null" { + return nil, nil + } + + var envelope struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return nil, fmt.Errorf("decode group details envelope: %w", err) + } + + switch envelope.Kind { + case GroupTypeStartup: + var g StartupGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeDeployment: + var g DeploymentGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypePromotion: + var g PromotionGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeTemporaryPermission: + var g TemporaryPermissionGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeIncidentResponse: + var g IncidentResponseGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + case GroupTypeCustom: + var g CustomGroup + if err := json.Unmarshal(raw, &g); err != nil { + return nil, err + } + return g, nil + default: + return nil, fmt.Errorf("unknown group kind %q", envelope.Kind) + } +} diff --git a/types/config_change_groups_test.go b/types/config_change_groups_test.go new file mode 100644 index 000000000..c0cdca5dd --- /dev/null +++ b/types/config_change_groups_test.go @@ -0,0 +1,117 @@ +package types + +import ( + "encoding/json" + "testing" + "time" + + "github.com/google/uuid" + "github.com/onsi/gomega" +) + +func TestGroupDetailsRoundTrip(t *testing.T) { + cfg := uuid.New() + grant := uuid.New() + revoke := uuid.New() + dur := int64(3600) + now := time.Now().UTC().Truncate(time.Second) + + cases := []struct { + name string + in GroupType + }{ + { + name: "startup", + in: StartupGroup{ConfigID: cfg, Reason: "CrashLoopBackOff", RestartCount: 3}, + }, + { + name: "deployment", + in: DeploymentGroup{ + Image: "registry/app:v1.2.3", + Version: "v1.2.3", + Commit: "abc123", + Strategy: "RollingUpdate", + TargetConfigIDs: []uuid.UUID{uuid.New(), uuid.New()}, + }, + }, + { + name: "promotion", + in: PromotionGroup{ + FromEnvironment: "dev", + ToEnvironment: "prod", + Version: "v1.2.3", + Artifact: "app", + PromotionChangeID: &grant, + TargetDeploymentIDs: []uuid.UUID{uuid.New()}, + }, + }, + { + name: "temporary_permission", + in: TemporaryPermissionGroup{ + UserID: "user-1", + RoleID: "admin", + Scope: "cluster-1", + GrantChangeID: &grant, + RevokeChangeID: &revoke, + DurationSeconds: &dur, + }, + }, + { + name: "incident_response", + in: IncidentResponseGroup{ + IncidentID: "INC-42", + OpenedAt: now, + ClosedAt: now.Add(time.Hour), + PlaybookRunIDs: []uuid.UUID{uuid.New()}, + }, + }, + { + name: "custom", + in: CustomGroup{Fields: map[string]any{"foo": "bar", "n": float64(42)}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + raw, err := json.Marshal(tc.in) + g.Expect(err).ToNot(gomega.HaveOccurred()) + + // envelope must carry the kind. + var envelope struct { + Kind string `json:"kind"` + } + g.Expect(json.Unmarshal(raw, &envelope)).To(gomega.Succeed()) + g.Expect(envelope.Kind).To(gomega.Equal(tc.in.Kind())) + + got, err := UnmarshalGroupDetails(raw) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got.Kind()).To(gomega.Equal(tc.in.Kind())) + + // Round-tripped value re-marshals to the same JSON. + reraw, err := json.Marshal(got) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(reraw).To(gomega.MatchJSON(raw)) + }) + } +} + +func TestUnmarshalGroupDetailsEmpty(t *testing.T) { + g := gomega.NewWithT(t) + + got, err := UnmarshalGroupDetails(nil) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.BeNil()) + + got, err = UnmarshalGroupDetails(json.RawMessage("null")) + g.Expect(err).ToNot(gomega.HaveOccurred()) + g.Expect(got).To(gomega.BeNil()) +} + +func TestUnmarshalGroupDetailsUnknownKind(t *testing.T) { + g := gomega.NewWithT(t) + + _, err := UnmarshalGroupDetails(json.RawMessage(`{"kind":"Nope/v1"}`)) + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("unknown group kind"))) +} diff --git a/types/config_changes.go b/types/config_changes.go new file mode 100644 index 000000000..49641de82 --- /dev/null +++ b/types/config_changes.go @@ -0,0 +1,732 @@ +package types + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" +) + +// ConfigChangeDetail is implemented by all typed detail structs +// for the ConfigChange.Details field. +type ConfigChangeDetail interface { + Kind() string +} + +var configChangeDetailTypes = []ConfigChangeDetail{ + UserChangeDetails{}, + ScreenshotDetails{}, + PermissionChangeDetails{}, + GroupMembership{}, + Identity{}, + Approval{}, + GitSource{}, + HelmSource{}, + ImageSource{}, + DatabaseSource{}, + Source{}, + Environment{}, + Event{}, + Test{}, + Promotion{}, + PipelineRun{}, + Change{}, + ConfigChange{}, + Restore{}, + Backup{}, + Dimension{}, + Scale{}, +} + +// Change type constants. +const ( + ChangeTypeCreate = "CREATE" + ChangeTypeUpdate = "UPDATE" + ChangeTypeDelete = "DELETE" + ChangeTypeDiff = "diff" + + ChangeTypeUserCreated = "UserCreated" + ChangeTypeUserDeleted = "UserDeleted" + ChangeTypeGroupMemberAdded = "GroupMemberAdded" + ChangeTypeGroupMemberRemoved = "GroupMemberRemoved" + + ChangeTypeScreenshot = "Screenshot" + + ChangeTypePermissionAdded = "PermissionAdded" + ChangeTypePermissionRemoved = "PermissionRemoved" + + ChangeTypeDeployment = "Deployment" + ChangeTypePromotion = "Promotion" + ChangeTypeApproved = "Approved" + ChangeTypeRejected = "Rejected" + ChangeTypeRollback = "Rollback" + + ChangeTypeBackupCompleted = "BackupCompleted" + ChangeTypeBackupRestored = "BackupRestored" + ChangeTypeBackupFailed = "BackupFailed" + + ChangeTypePipelineRunStarted = "PipelineRunStarted" + ChangeTypePipelineRunCompleted = "PipelineRunCompleted" + ChangeTypePipelineRunFailed = "PipelineRunFailed" + + ChangeTypeScaling = "Scaling" + + ChangeTypeCertificateRenewed = "CertificateRenewed" + ChangeTypeCertificateExpired = "CertificateExpired" + + ChangeTypeCostChange = "CostChange" + + ChangeTypePlaybookStarted = "PlaybookStarted" + ChangeTypePlaybookCompleted = "PlaybookCompleted" + ChangeTypePlaybookFailed = "PlaybookFailed" + + ChangeTypeRunInstances = "RunInstances" + ChangeTypeRegisterNode = "RegisterNode" + ChangeTypePulled = "Pulled" +) + +func marshalWithKind(kind string, v any) ([]byte, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + if len(data) > 2 { + return append([]byte(`{"kind":"`+kind+`",`), data[1:]...), nil + } + return []byte(`{"kind":"` + kind + `"}`), nil +} + +func pointerIfNotZero[T any](v T) *T { + if reflect.ValueOf(v).IsZero() { + return nil + } + return &v +} + +// UnmarshalChangeDetails unmarshals the details field of a ConfigChange into the appropriate typed struct based on the "kind" field. +func UnmarshalChangeDetails(data []byte) (ConfigChangeDetail, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nil, nil + } + + var envelope struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(trimmed, &envelope); err != nil { + return nil, fmt.Errorf("decode config change details envelope: %w", err) + } + + for _, candidate := range configChangeDetailTypes { + if candidate.Kind() != envelope.Kind { + continue + } + + value := reflect.New(reflect.TypeOf(candidate)) + if err := json.Unmarshal(trimmed, value.Interface()); err != nil { + return nil, err + } + + detail, ok := value.Elem().Interface().(ConfigChangeDetail) + if !ok { + return nil, fmt.Errorf("decoded config change detail %q does not implement ConfigChangeDetail", envelope.Kind) + } + + return detail, nil + } + + return nil, fmt.Errorf("unknown config change detail kind %q", envelope.Kind) +} + +type UserChangeDetails struct { + UserID string `json:"user_id,omitempty"` + UserName string `json:"user_name,omitempty"` + UserEmail string `json:"user_email,omitempty"` + UserType string `json:"user_type,omitempty"` + GroupID string `json:"group_id,omitempty"` + GroupName string `json:"group_name,omitempty"` + Tenant string `json:"tenant,omitempty"` +} + +func (d UserChangeDetails) Kind() string { return "UserChange/v1" } +func (d UserChangeDetails) MarshalJSON() ([]byte, error) { + type raw UserChangeDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type ScreenshotDetails struct { + ArtifactID string `json:"artifact_id,omitempty"` + URL string `json:"url,omitempty"` + ContentType string `json:"content_type,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` +} + +func (d ScreenshotDetails) Kind() string { return "Screenshot/v1" } +func (d ScreenshotDetails) MarshalJSON() ([]byte, error) { + type raw ScreenshotDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type PermissionChangeDetails struct { + UserID string `json:"user_id,omitempty"` + UserName string `json:"user_name,omitempty"` + GroupID string `json:"group_id,omitempty"` + GroupName string `json:"group_name,omitempty"` + RoleID string `json:"role_id,omitempty"` + RoleName string `json:"role_name,omitempty"` + RoleType string `json:"role_type,omitempty"` + Scope string `json:"scope,omitempty"` +} + +func (d PermissionChangeDetails) Kind() string { return "PermissionChange/v1" } +func (d PermissionChangeDetails) MarshalJSON() ([]byte, error) { + type raw PermissionChangeDetails + return marshalWithKind(d.Kind(), raw(d)) +} + +type GroupMembershipAction string + +const ( + GroupMembershipActionAdded GroupMembershipAction = "Added" + GroupMembershipActionRemoved GroupMembershipAction = "Removed" +) + +type GroupMembership struct { + Group Identity `json:"group,omitempty"` + Member Identity `json:"member,omitempty"` + Action GroupMembershipAction `json:"action,omitempty"` + Tenant string `json:"tenant,omitempty"` +} + +func (d GroupMembership) Kind() string { return "GroupMembership/v1" } +func (d GroupMembership) MarshalJSON() ([]byte, error) { + type payload struct { + Group *Identity `json:"group,omitempty"` + Member *Identity `json:"member,omitempty"` + Action GroupMembershipAction `json:"action,omitempty"` + Tenant string `json:"tenant,omitempty"` + } + return marshalWithKind(d.Kind(), payload{ + Group: pointerIfNotZero(d.Group), + Member: pointerIfNotZero(d.Member), + Action: d.Action, + Tenant: d.Tenant, + }) +} + +type DeploymentType string + +const ( + ImageUpgrade DeploymentType = "ImageUpgrade" + ImageDowngrade DeploymentType = "ImageDowngrade" + Rollout DeploymentType = "Rollout" + Restart DeploymentType = "Restart" + Rollback DeploymentType = "Rollback" + ConfigurationChange DeploymentType = "ConfigurationChange" + ScaleUp DeploymentType = "ScaleUp" + ScaleDown DeploymentType = "ScaleDown" + ScaleIn DeploymentType = "ScaleIn" + ScaleOut DeploymentType = "ScaleOut" + PolicyChange DeploymentType = "PolicyChange" + SchemaChange DeploymentType = "SchemaChange" + DataMigration DeploymentType = "DataMigration" + DataFix DeploymentType = "DataFix" + OtherDeploymentChange DeploymentType = "Other" +) + +type IdentityType string + +const ( + IdentityTypeUser IdentityType = "User" + IdentityTypeGroup IdentityType = "Group" + IdentityTypeRole IdentityType = "Role" + IdentityTypeCI IdentityType = "System:CI" + IdentityTypeAuto IdentityType = "System:Auto" + IdentityTypeScan IdentityType = "System:Scan" + IdentityTypeTest IdentityType = "System:Test" + IdentityTypeCanary IdentityType = "System:Canary" +) + +type ApprovalStage string + +const ( + ApprovalStagePreDeployment ApprovalStage = "PreDeployment" + ApprovalStagePostDeployment ApprovalStage = "PostDeployment" + ApprovalStagePrePromotion ApprovalStage = "PrePromotion" + ApprovalStagePostPromotion ApprovalStage = "PostPromotion" + ApprovalStageManual ApprovalStage = "Manual" + ApprovalStageAutomated ApprovalStage = "Automated" +) + +type ApprovalStatus string + +const ( + ApprovalStatusApproved ApprovalStatus = "Approved" + ApprovalStatusRejected ApprovalStatus = "Rejected" + ApprovalStatusPending ApprovalStatus = "Pending" + ApprovalStatusExpired ApprovalStatus = "Expired" +) + +type Identity struct { + ID string `json:"id,omitempty"` + Type IdentityType `json:"type,omitempty"` + // Optional human-readable name for the identity, e.g. user name or group name. Not required if ID is present and meaningful on its own. + Name string `json:"name,omitempty"` + // Optional comment about the identity, e.g. reason for approval/rejection, or details about the change. + Comment string `json:"comment,omitempty"` +} + +func (i Identity) IsEmpty() bool { + return i.ID == "" && i.Type == "" && i.Name == "" +} + +func (i Identity) Kind() string { return "Identity/v1" } +func (i Identity) MarshalJSON() ([]byte, error) { + type raw Identity + return marshalWithKind(i.Kind(), raw(i)) +} + +type Approval struct { + Event `json:",inline"` + // Optional identity of the person or system that submitted the approval request. May be empty for automated approvals or when the submitter is unknown. + SubmittedBy *Identity `json:"submitted_by,omitempty"` + // Optional identity of the person or system that approved or rejected the change. May be empty if the approval is still pending or if the approver is unknown. + Approver *Identity `json:"approver,omitempty"` + Stage ApprovalStage `json:"stage,omitempty"` + Status ApprovalStatus `json:"status,omitempty"` +} + +func (a Approval) Kind() string { return "Approval/v1" } +func (a Approval) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + SubmittedBy *Identity `json:"submitted_by,omitempty"` + Approver *Identity `json:"approver,omitempty"` + Stage ApprovalStage `json:"stage,omitempty"` + Status ApprovalStatus `json:"status,omitempty"` + } + return marshalWithKind(a.Kind(), payload{ + rawEvent: rawEvent(a.Event), + SubmittedBy: a.SubmittedBy, + Approver: a.Approver, + Stage: a.Stage, + Status: a.Status, + }) +} + +type GitSource struct { + URL string `json:"url,omitempty"` + Branch string `json:"branch,omitempty"` + CommitSHA string `json:"commit_sha,omitempty"` + Version string `json:"version,omitempty"` + Tags string `json:"tags,omitempty"` +} + +func (s GitSource) Kind() string { return "GitSource/v1" } +func (s GitSource) MarshalJSON() ([]byte, error) { + type raw GitSource + return marshalWithKind(s.Kind(), raw(s)) +} + +type HelmSource struct { + ChartName string `json:"chart_name,omitempty"` + ChartVersion string `json:"chart_version,omitempty"` + RepoURL string `json:"repo_url,omitempty"` +} + +func (s HelmSource) Kind() string { return "HelmSource/v1" } +func (s HelmSource) MarshalJSON() ([]byte, error) { + type raw HelmSource + return marshalWithKind(s.Kind(), raw(s)) +} + +type ImageSource struct { + Registry string `json:"registry,omitempty"` + ImageName string `json:"image,omitempty"` + Version string `json:"version,omitempty"` + SHA string `json:"sha,omitempty"` +} + +func (s ImageSource) Kind() string { return "ImageSource/v1" } +func (s ImageSource) MarshalJSON() ([]byte, error) { + type raw ImageSource + return marshalWithKind(s.Kind(), raw(s)) +} + +type DatabaseSource struct { + // Database type, e.g. "PostgreSQL", "MySQL", "MongoDB" + Type string `json:"type,omitempty"` + // Database name, e.g. "mydb" + Name string `json:"name,omitempty"` + // Schema name, e.g. "public" + SchemaName string `json:"schema,omitempty"` + // Database version, e.g. "12.3" + Version string `json:"version,omitempty"` + // Server or cluster endpoint, e.g. "mydb.cluster-123.us-east-1.rds.amazonaws.com:5432" + Endpoint string `json:"endpoint,omitempty"` +} + +func (s DatabaseSource) Kind() string { return "DatabaseSource/v1" } +func (s DatabaseSource) MarshalJSON() ([]byte, error) { + type raw DatabaseSource + return marshalWithKind(s.Kind(), raw(s)) +} + +type Source struct { + Git *GitSource `json:"git,omitempty"` + Helm *HelmSource `json:"helm,omitempty"` + Image *ImageSource `json:"image,omitempty"` + Database *DatabaseSource `json:"database,omitempty"` + KustomizationSource *GitSource `json:"kustomization,omitempty"` + ArgocdSource *GitSource `json:"argocd,omitempty"` + OtherSource *string `json:"other,omitempty"` + Path string `json:"path,omitempty"` // Optional path within the source, e.g. file path in git repo or chart path in Helm repo +} + +func (s Source) Kind() string { return "Source/v1" } +func (s Source) MarshalJSON() ([]byte, error) { + type raw Source + return marshalWithKind(s.Kind(), raw(s)) +} + +type EnvironmentType string +type EnvironmentStage string + +const ( + EnvironmentStageDevelopment EnvironmentStage = "Development" + EnvironmentStageStaging EnvironmentStage = "Staging" + EnvironmentStageProduction EnvironmentStage = "Production" + EnvironmentStageUAT EnvironmentStage = "UAT" + EnvironmentStageQA EnvironmentStage = "QA" + + EnvironmentTypeKubernetes EnvironmentType = "Kubernetes" + EnvironmentTypeCloud EnvironmentType = "Cloud" + EnvironmentTypeOnPrem EnvironmentType = "On-Premises" + EnvironmentTypeOther EnvironmentType = "Other" +) + +type Environment struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + // Optional type of environment, e.g. Kubernetes, Cloud, On-Premises, etc. + EnvironmentType EnvironmentType `json:"type,omitempty"` + // Optional stage or lifecycle phase of the environment, e.g. Development, Staging, Production, UAT, QA, etc. + Stage EnvironmentStage `json:"stage,omitempty"` + + Identifier string `json:"identifier,omitempty"` + // Optional tags for additional metadata, e.g. team, cost center, owner, cluster/namespace + Tags map[string]string `json:"tags,omitempty"` +} + +func (e Environment) Kind() string { return "Environment/v1" } +func (e Environment) MarshalJSON() ([]byte, error) { + type raw Environment + return marshalWithKind(e.Kind(), raw(e)) +} + +type TestingType string +type TestingStatus string +type TestingResult string +type Status string + +const ( + TestingTypeUnit TestingType = "Unit" + TestingTypeIntegration TestingType = "Integration" + TestingTypeE2E TestingType = "End-to-End" + TestingTypePerformance TestingType = "Performance" + TestingTypeSecurity TestingType = "Security" + + TestingStatusPending TestingStatus = "Pending" + TestingStatusRunning TestingStatus = "Running" + TestingStatusPassed TestingStatus = "Passed" + TestingStatusFailed TestingStatus = "Failed" + TestingStatusSkipped TestingStatus = "Skipped" + TestingStatusError TestingStatus = "Error" + + StatusPending Status = "Pending" + StatusRunning Status = "Running" + StatusTimeout Status = "Timeout" + StatusCompleted Status = "Completed" + StatusFailed Status = "Failed" + StatusApproved Status = "Approved" + StatusRejected Status = "Rejected" + + TestingResultFlaky TestingResult = "Flaky" + TestingResultFailed TestingResult = "Failed" + TestingResultPassed TestingResult = "Passed" +) + +type Event struct { + ID string `json:"id,omitempty"` + URL string `json:"url,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Properties map[string]string `json:"properties,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + +func (e Event) Kind() string { return "Event/v1" } +func (e Event) MarshalJSON() ([]byte, error) { + type raw Event + return marshalWithKind(e.Kind(), raw(e)) +} + +type Test struct { + Event `json:",inline"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Type TestingType `json:"type,omitempty"` + Status TestingStatus `json:"status,omitempty"` + Result TestingResult `json:"result,omitempty"` +} + +func (t Test) Kind() string { return "Test/v1" } +func (t Test) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Type TestingType `json:"type,omitempty"` + Status TestingStatus `json:"status,omitempty"` + Result TestingResult `json:"result,omitempty"` + } + return marshalWithKind(t.Kind(), payload{ + rawEvent: rawEvent(t.Event), + Name: t.Name, + Description: t.Description, + Type: t.Type, + Status: t.Status, + Result: t.Result, + }) +} + +type Promotion struct { + Event `json:",inline"` + + // Optional source and target environments for the promotion. If not specified, the promotion is assumed to be within the same environment. + From Environment `json:"from,omitempty"` + To Environment `json:"to,omitempty"` + // Optional source for the promotion, e.g. Git repo, Helm chart, container image, database schema, etc. + Source Source `json:"source,omitempty"` + // Optional version or identifier for the promoted artifact, e.g. image tag, chart version, git commit, database schema version, etc. + Version string `json:"version,omitempty"` + // Optional list of identities who approved the promotion, e.g. users or groups who approved the change, or CI systems that ran tests and checks. + Approvals []Approval `json:"approvals,omitempty"` + // + Artifact string `json:"artifact,omitempty"` +} + +func (p Promotion) Kind() string { return "Promotion/v1" } +func (p Promotion) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + From *Environment `json:"from,omitempty"` + To *Environment `json:"to,omitempty"` + Source *Source `json:"source,omitempty"` + Version string `json:"version,omitempty"` + Approvals []Approval `json:"approvals,omitempty"` + Artifact string `json:"artifact,omitempty"` + } + return marshalWithKind(p.Kind(), payload{ + rawEvent: rawEvent(p.Event), + From: pointerIfNotZero(p.From), + To: pointerIfNotZero(p.To), + Source: pointerIfNotZero(p.Source), + Version: p.Version, + Approvals: p.Approvals, + Artifact: p.Artifact, + }) +} + +type PipelineRun struct { + Event `json:",inline"` + Environment Environment `json:"environment,omitempty"` + Status Status `json:"status,omitempty"` +} + +func (p PipelineRun) Kind() string { return "PipelineRun/v1" } +func (p PipelineRun) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + Environment *Environment `json:"environment,omitempty"` + Status Status `json:"status,omitempty"` + } + return marshalWithKind(p.Kind(), payload{ + rawEvent: rawEvent(p.Event), + Environment: pointerIfNotZero(p.Environment), + Status: p.Status, + }) +} + +type Change struct { + Path string `json:"path,omitempty"` + From map[string]any `json:"from,omitempty"` + To map[string]any `json:"to,omitempty"` + Type string `json:"type,omitempty"` +} + +func (c Change) Kind() string { return "Change/v1" } +func (c Change) MarshalJSON() ([]byte, error) { + type raw Change + return marshalWithKind(c.Kind(), raw(c)) +} + +type ConfigChange struct { + Event `json:",inline"` + Author Identity `json:"author,omitempty"` + Changes []Change `json:"changes,omitempty"` + Environment Environment `json:"environment,omitempty"` + Source Source `json:"source,omitempty"` +} + +func (c ConfigChange) Kind() string { return "ConfigChange/v1" } +func (c ConfigChange) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + Author *Identity `json:"author,omitempty"` + Changes []Change `json:"changes,omitempty"` + Environment *Environment `json:"environment,omitempty"` + Source *Source `json:"source,omitempty"` + } + return marshalWithKind(c.Kind(), payload{ + rawEvent: rawEvent(c.Event), + Author: pointerIfNotZero(c.Author), + Changes: c.Changes, + Environment: pointerIfNotZero(c.Environment), + Source: pointerIfNotZero(c.Source), + }) +} + +type BackupType string + +const ( + BackupTypeDump BackupType = "Dump" + BackupTypeSnapshot BackupType = "Snapshot" + BackupTypeStorageBackup BackupType = "StorageBackup" + BackupTypeOffsite BackupType = "Offsite" + BackupTypeOffAccount BackupType = "OffAccount" + BackupTypeOffRegion BackupType = "OffRegion" +) + +type RestoreType string + +const ( + RestoreTypeClone RestoreType = "Clone" + RestoreTypeSnapshotReset RestoreType = "SnapshotReset" + RestoreTypePointInTime RestoreType = "PointInTime" + RestoreTypeDisasterRecovery RestoreType = "DisasterRecovery" + RestoreTypeTest RestoreType = "RestoreTest" + RestoreTypeData RestoreType = "Data" +) + +type Restore struct { + Event `json:",inline"` + // Optional source and target environments for the restore. If not specified, the restore is assumed to be within the same environment. + From Environment `json:"from,omitempty"` + To Environment `json:"to,omitempty"` + // Optional source for the restore, e.g. Git repo, Helm chart, container image, database schema, etc. + Source Source `json:"source,omitempty"` + // Optional version or identifier for the restored artifact, e.g. image tag, chart version, git commit, database schema version, etc. + Status Status `json:"status,omitempty"` +} + +func (r Restore) Kind() string { return "Restore/v1" } +func (r Restore) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + From *Environment `json:"from,omitempty"` + To *Environment `json:"to,omitempty"` + Source *Source `json:"source,omitempty"` + Status Status `json:"status,omitempty"` + } + return marshalWithKind(r.Kind(), payload{ + rawEvent: rawEvent(r.Event), + From: pointerIfNotZero(r.From), + To: pointerIfNotZero(r.To), + Source: pointerIfNotZero(r.Source), + Status: r.Status, + }) +} + +type Backup struct { + BackupType BackupType `json:"backup_type,omitempty"` + CreatedBy Identity `json:"created_by,omitempty"` + Environment Environment `json:"environment,omitempty"` + Event `json:",inline"` + EndTimestamp string `json:"end,omitempty"` + Status Status `json:"status,omitempty"` + Size string `json:"size,omitempty"` + Delta string `json:"delta,omitempty"` +} + +func (d Backup) Kind() string { return "Backup/v1" } +func (d Backup) MarshalJSON() ([]byte, error) { + type rawEvent Event + type payload struct { + rawEvent `json:",inline"` + BackupType BackupType `json:"backup_type,omitempty"` + CreatedBy *Identity `json:"created_by,omitempty"` + Environment *Environment `json:"environment,omitempty"` + EndTimestamp string `json:"end,omitempty"` + Status Status `json:"status,omitempty"` + Size string `json:"size,omitempty"` + Delta string `json:"delta,omitempty"` + } + return marshalWithKind(d.Kind(), payload{ + rawEvent: rawEvent(d.Event), + BackupType: d.BackupType, + CreatedBy: pointerIfNotZero(d.CreatedBy), + Environment: pointerIfNotZero(d.Environment), + EndTimestamp: d.EndTimestamp, + Status: d.Status, + Size: d.Size, + Delta: d.Delta, + }) +} + +type ScalingDimension string + +const ( + ScalingDimensionCPU ScalingDimension = "CPU" + ScalingDimensionMemory ScalingDimension = "Memory" + ScalingDimensionReplicas ScalingDimension = "Replicas" + ScalingDimensionCustom ScalingDimension = "Custom" +) + +type Dimension struct { + Min string `json:"min,omitempty"` + Max string `json:"max,omitempty"` + Desired string `json:"desired,omitempty"` +} + +func (d Dimension) Kind() string { return "Dimension/v1" } +func (d Dimension) MarshalJSON() ([]byte, error) { + type raw Dimension + return marshalWithKind(d.Kind(), raw(d)) +} + +type Scale struct { + Dimension ScalingDimension `json:"dimension,omitempty"` + PreviousValue Dimension `json:"previous_value,omitempty"` + Value Dimension `json:"value,omitempty"` +} + +func (d Scale) Kind() string { return "Scale/v1" } +func (d Scale) MarshalJSON() ([]byte, error) { + type payload struct { + Dimension ScalingDimension `json:"dimension,omitempty"` + PreviousValue *Dimension `json:"previous_value,omitempty"` + Value *Dimension `json:"value,omitempty"` + } + return marshalWithKind(d.Kind(), payload{ + Dimension: d.Dimension, + PreviousValue: pointerIfNotZero(d.PreviousValue), + Value: pointerIfNotZero(d.Value), + }) +} diff --git a/types/config_changes_test.go b/types/config_changes_test.go new file mode 100644 index 000000000..952c266be --- /dev/null +++ b/types/config_changes_test.go @@ -0,0 +1,369 @@ +package types + +import ( + "encoding/json" + "slices" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type kinded interface { + Kind() string +} + +func expectKind(v kinded) map[string]any { + data, err := json.Marshal(v) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal(v.Kind())) + return m +} + +var _ = Describe("ConfigChangeDetails", func() { + DescribeTable("current detail payloads implement ConfigChangeDetail", + func(d ConfigChangeDetail, expectedFields map[string]any) { + m := expectKind(d) + for key, val := range expectedFields { + Expect(m[key]).To(Equal(val)) + } + }, + Entry("UserChange", UserChangeDetails{UserName: "alice"}, map[string]any{ + "user_name": "alice", + }), + Entry("Screenshot", ScreenshotDetails{URL: "https://example.com"}, map[string]any{ + "url": "https://example.com", + }), + Entry("PermissionChange", PermissionChangeDetails{RoleName: "admin"}, map[string]any{ + "role_name": "admin", + }), + Entry("GroupMembership", GroupMembership{ + Group: Identity{Name: "platform-admins", Type: IdentityTypeGroup}, + Member: Identity{Name: "alice", Type: IdentityTypeUser}, + Action: GroupMembershipActionAdded, + }, map[string]any{ + "action": string(GroupMembershipActionAdded), + }), + Entry("Backup", Backup{Status: StatusCompleted, Size: "4.2GB"}, map[string]any{ + "status": string(StatusCompleted), + "size": "4.2GB", + }), + Entry("Scale", Scale{ + Dimension: ScalingDimensionReplicas, + PreviousValue: Dimension{Desired: "2"}, + Value: Dimension{Desired: "5"}, + }, map[string]any{ + "dimension": string(ScalingDimensionReplicas), + }), + ) + + DescribeTable("exported structs inject kind", + func(v kinded, expectedFields map[string]any) { + m := expectKind(v) + for key, val := range expectedFields { + Expect(m[key]).To(Equal(val)) + } + }, + Entry("Identity", Identity{Name: "alice"}, map[string]any{ + "name": "alice", + }), + Entry("Approval", Approval{ + Event: Event{ID: "evt-1"}, + SubmittedBy: &Identity{Name: "submitter"}, + Approver: &Identity{Name: "approver"}, + Stage: ApprovalStageManual, + Status: ApprovalStatusApproved, + }, map[string]any{ + "id": "evt-1", + "stage": string(ApprovalStageManual), + "status": string(ApprovalStatusApproved), + }), + Entry("GitSource", GitSource{URL: "https://example.com/repo.git"}, map[string]any{ + "url": "https://example.com/repo.git", + }), + Entry("HelmSource", HelmSource{ChartName: "api"}, map[string]any{ + "chart_name": "api", + }), + Entry("ImageSource", ImageSource{ImageName: "backend"}, map[string]any{ + "image": "backend", + }), + Entry("DatabaseSource", DatabaseSource{Name: "appdb"}, map[string]any{ + "name": "appdb", + }), + Entry("Source", Source{ + Git: &GitSource{URL: "https://example.com/repo.git"}, + Path: "deploy/app", + }, map[string]any{ + "path": "deploy/app", + }), + Entry("Environment", Environment{Name: "prod"}, map[string]any{ + "name": "prod", + }), + Entry("Event", Event{ID: "evt-2"}, map[string]any{ + "id": "evt-2", + }), + Entry("Test", Test{ + Event: Event{ID: "evt-3"}, + Name: "smoke", + Type: TestingTypeE2E, + Status: TestingStatusPassed, + Result: TestingResultPassed, + }, map[string]any{ + "id": "evt-3", + "name": "smoke", + "type": string(TestingTypeE2E), + "status": string(TestingStatusPassed), + "result": string(TestingResultPassed), + }), + Entry("Promotion", Promotion{ + Event: Event{ID: "evt-4"}, + From: Environment{Name: "staging"}, + To: Environment{Name: "prod"}, + Source: Source{Git: &GitSource{URL: "https://example.com/repo.git"}}, + Version: "v1.2.3", + }, map[string]any{ + "id": "evt-4", + "version": "v1.2.3", + }), + Entry("PipelineRun", PipelineRun{ + Event: Event{ID: "evt-5"}, + Environment: Environment{Name: "prod"}, + Status: StatusRunning, + }, map[string]any{ + "id": "evt-5", + "status": string(StatusRunning), + }), + Entry("Change", Change{ + Path: ".spec.replicas", + From: map[string]any{"desired": "2"}, + To: map[string]any{"desired": "3"}, + Type: "update", + }, map[string]any{ + "path": ".spec.replicas", + "type": "update", + }), + Entry("ConfigChange", ConfigChange{ + Event: Event{ID: "evt-6"}, + Author: Identity{Name: "alice"}, + Changes: []Change{{Path: ".spec.replicas", Type: "update"}}, + Environment: Environment{Name: "prod"}, + Source: Source{Git: &GitSource{URL: "https://example.com/repo.git"}}, + }, map[string]any{ + "id": "evt-6", + }), + Entry("Restore", Restore{ + Event: Event{ID: "evt-7"}, + From: Environment{Name: "backup"}, + To: Environment{Name: "prod"}, + Source: Source{Database: &DatabaseSource{Name: "appdb"}}, + Status: StatusCompleted, + }, map[string]any{ + "id": "evt-7", + "status": string(StatusCompleted), + }), + Entry("Dimension", Dimension{Desired: "3"}, map[string]any{ + "desired": "3", + }), + ) + + It("preserves embedded event fields and nested kinds", func() { + data, err := json.Marshal(ConfigChange{ + Event: Event{ID: "evt-8", Timestamp: "2026-04-10T12:00:00Z"}, + Author: Identity{Name: "alice"}, + Changes: []Change{{ + Path: ".spec.replicas", + Type: "update", + }}, + Environment: Environment{Name: "prod", Stage: EnvironmentStageProduction}, + Source: Source{ + Git: &GitSource{ + URL: "https://example.com/repo.git", + }, + }, + }) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("ConfigChange/v1")) + Expect(m["id"]).To(Equal("evt-8")) + Expect(m["timestamp"]).To(Equal("2026-04-10T12:00:00Z")) + + author := m["author"].(map[string]any) + Expect(author["kind"]).To(Equal("Identity/v1")) + Expect(author["name"]).To(Equal("alice")) + + environment := m["environment"].(map[string]any) + Expect(environment["kind"]).To(Equal("Environment/v1")) + Expect(environment["name"]).To(Equal("prod")) + + source := m["source"].(map[string]any) + Expect(source["kind"]).To(Equal("Source/v1")) + git := source["git"].(map[string]any) + Expect(git["kind"]).To(Equal("GitSource/v1")) + + changes := m["changes"].([]any) + Expect(changes).To(HaveLen(1)) + change := changes[0].(map[string]any) + Expect(change["kind"]).To(Equal("Change/v1")) + Expect(change["path"]).To(Equal(".spec.replicas")) + }) + + It("emits nested identity kinds for GroupMembership", func() { + data, err := json.Marshal(GroupMembership{ + Group: Identity{ID: "g-1", Name: "platform-admins", Type: IdentityTypeGroup}, + Member: Identity{ID: "u-1", Name: "alice", Type: IdentityTypeUser}, + Action: GroupMembershipActionAdded, + Tenant: "acme", + }) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("GroupMembership/v1")) + Expect(m["action"]).To(Equal(string(GroupMembershipActionAdded))) + Expect(m["tenant"]).To(Equal("acme")) + + group := m["group"].(map[string]any) + Expect(group["kind"]).To(Equal("Identity/v1")) + Expect(group["name"]).To(Equal("platform-admins")) + Expect(group["type"]).To(Equal(string(IdentityTypeGroup))) + + member := m["member"].(map[string]any) + Expect(member["kind"]).To(Equal("Identity/v1")) + Expect(member["name"]).To(Equal("alice")) + Expect(member["type"]).To(Equal(string(IdentityTypeUser))) + }) + + It("omits zero-value nested struct fields from outer payloads", func() { + data, err := json.Marshal(PipelineRun{ + Event: Event{ID: "evt-9"}, + Status: StatusPending, + }) + Expect(err).ToNot(HaveOccurred()) + + var m map[string]any + Expect(json.Unmarshal(data, &m)).To(Succeed()) + Expect(m["kind"]).To(Equal("PipelineRun/v1")) + Expect(m["id"]).To(Equal("evt-9")) + Expect(m).ToNot(HaveKey("environment")) + }) + + It("registers all exported structs for kind lookup", func() { + kinds := make([]string, 0, len(configChangeDetailTypes)) + for _, candidate := range configChangeDetailTypes { + kinds = append(kinds, candidate.Kind()) + } + slices.Sort(kinds) + + Expect(kinds).To(Equal([]string{ + "Approval/v1", + "Backup/v1", + "Change/v1", + "ConfigChange/v1", + "DatabaseSource/v1", + "Dimension/v1", + "Environment/v1", + "Event/v1", + "GitSource/v1", + "GroupMembership/v1", + "HelmSource/v1", + "Identity/v1", + "ImageSource/v1", + "PermissionChange/v1", + "PipelineRun/v1", + "Promotion/v1", + "Restore/v1", + "Scale/v1", + "Screenshot/v1", + "Source/v1", + "Test/v1", + "UserChange/v1", + })) + }) + + DescribeTable("UnmarshalChangeDetails returns the matching registered type", + func(in ConfigChangeDetail, expected any) { + raw, err := json.Marshal(in) + Expect(err).ToNot(HaveOccurred()) + + got, err := UnmarshalChangeDetails(raw) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeAssignableToTypeOf(expected)) + + reraw, err := json.Marshal(got) + Expect(err).ToNot(HaveOccurred()) + Expect(reraw).To(MatchJSON(raw)) + }, + Entry("UserChange", UserChangeDetails{UserName: "alice"}, UserChangeDetails{}), + Entry("Screenshot", ScreenshotDetails{URL: "https://example.com"}, ScreenshotDetails{}), + Entry("PermissionChange", PermissionChangeDetails{RoleName: "admin"}, PermissionChangeDetails{}), + Entry("GroupMembership", GroupMembership{ + Group: Identity{ID: "g-1", Type: IdentityTypeGroup}, + Member: Identity{ID: "u-1", Type: IdentityTypeUser}, + Action: GroupMembershipActionRemoved, + }, GroupMembership{}), + Entry("Identity", Identity{Name: "alice"}, Identity{}), + Entry("Approval", Approval{ + Event: Event{ID: "evt-10"}, + SubmittedBy: &Identity{Name: "alice"}, + Status: ApprovalStatusApproved, + }, Approval{}), + Entry("GitSource", GitSource{URL: "https://example.com/repo.git"}, GitSource{}), + Entry("HelmSource", HelmSource{ChartName: "api"}, HelmSource{}), + Entry("ImageSource", ImageSource{ImageName: "backend"}, ImageSource{}), + Entry("DatabaseSource", DatabaseSource{Name: "appdb"}, DatabaseSource{}), + Entry("Source", Source{ + Git: &GitSource{URL: "https://example.com/repo.git"}, + Path: "deploy/app", + }, Source{}), + Entry("Environment", Environment{Name: "prod"}, Environment{}), + Entry("Event", Event{ID: "evt-11"}, Event{}), + Entry("Test", Test{ + Event: Event{ID: "evt-12"}, + Name: "smoke", + }, Test{}), + Entry("Promotion", Promotion{ + Event: Event{ID: "evt-13"}, + Version: "v1.2.3", + }, Promotion{}), + Entry("PipelineRun", PipelineRun{ + Event: Event{ID: "evt-14"}, + Status: StatusRunning, + }, PipelineRun{}), + Entry("Change", Change{Path: ".spec.replicas", Type: "update"}, Change{}), + Entry("ConfigChange", ConfigChange{ + Event: Event{ID: "evt-15"}, + Author: Identity{Name: "alice"}, + }, ConfigChange{}), + Entry("Restore", Restore{ + Event: Event{ID: "evt-16"}, + Status: StatusCompleted, + }, Restore{}), + Entry("Backup", Backup{Status: StatusCompleted, Size: "4.2GB"}, Backup{}), + Entry("Dimension", Dimension{Desired: "3"}, Dimension{}), + Entry("Scale", Scale{Dimension: ScalingDimensionReplicas, Value: Dimension{Desired: "3"}}, Scale{}), + ) + + It("returns nil for empty and null details payloads", func() { + got, err := UnmarshalChangeDetails(nil) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeNil()) + + got, err = UnmarshalChangeDetails([]byte("null")) + Expect(err).ToNot(HaveOccurred()) + Expect(got).To(BeNil()) + }) + + It("returns an error for an unknown kind", func() { + _, err := UnmarshalChangeDetails([]byte(`{"kind":"Nope/v1"}`)) + Expect(err).To(MatchError(ContainSubstring("unknown config change detail kind"))) + }) + + It("returns an error for invalid JSON", func() { + _, err := UnmarshalChangeDetails([]byte(`{`)) + Expect(err).To(MatchError(ContainSubstring("decode config change details envelope"))) + }) +}) diff --git a/types/resource_selector.go b/types/resource_selector.go index 62ec890e2..602f6b264 100644 --- a/types/resource_selector.go +++ b/types/resource_selector.go @@ -86,6 +86,29 @@ type ResourceSelector struct { Statuses Items `yaml:"statuses,omitempty" json:"statuses,omitempty"` } +// Expand splits a semicolon-delimited Search into multiple ResourceSelectors, +// each representing an independent query whose results are unioned. +func (rs ResourceSelector) Expand() []ResourceSelector { + if !strings.Contains(rs.Search, ";") { + return []ResourceSelector{rs} + } + segments := strings.Split(rs.Search, ";") + var result []ResourceSelector + for _, seg := range segments { + seg = strings.TrimSpace(seg) + if seg == "" { + continue + } + expanded := rs + expanded.Search = seg + result = append(result, expanded) + } + if len(result) == 0 { + return []ResourceSelector{rs} + } + return result +} + // ParseFilteringQuery parses a filtering query string. // It returns four slices: 'in', 'notIN', 'prefix', and 'suffix'. func ParseFilteringQuery(query string, decodeURL bool) (in []interface{}, notIN []interface{}, prefix, suffix []string, err error) { diff --git a/views/030_config_changes.sql b/views/030_config_changes.sql index c2a217e7a..347f886cd 100644 --- a/views/030_config_changes.sql +++ b/views/030_config_changes.sql @@ -27,7 +27,8 @@ BEGIN patches = NEW.patches, severity = NEW.severity, source = NEW.source, - summary = NEW.summary + summary = NEW.summary, + group_id = NEW.group_id WHERE id = NEW.id; diff --git a/views/038_config_access.sql b/views/038_config_access.sql index 02aa2826e..74e43bd2e 100644 --- a/views/038_config_access.sql +++ b/views/038_config_access.sql @@ -11,7 +11,7 @@ SELECT config_access.config_id, external_user_groups.external_user_id, config_access.external_group_id AS external_group_id, - NULL AS external_role_id, + config_access.external_role_id, config_access.created_at, config_access.deleted_at, config_access.deleted_by, diff --git a/views/045_merge_external_entities.sql b/views/045_merge_external_entities.sql index 96a908317..a2e03155e 100644 --- a/views/045_merge_external_entities.sql +++ b/views/045_merge_external_entities.sql @@ -1,12 +1,74 @@ +-- _debug_log: generic debug helper for instrumenting plpgsql functions. +-- Gated on session setting `debug_log.enabled = 'on'`. Emits the step + +-- detail as a `RAISE NOTICE` so the output survives savepoint rollback +-- (NOTICE messages stream to the client out-of-band and are not rolled +-- back by transaction/savepoint aborts — unlike INSERTs into any table, +-- temp or otherwise). +-- +-- The NOTICE payload is prefixed with a sentinel `_DEBUG_LOG:` so callers +-- (both psql and the Go dump helper) can grep for it among other notices. +-- +-- Usage: +-- SET LOCAL debug_log.enabled = 'on'; +-- ... call the instrumented function inside a savepoint ... +-- -- NOTICE lines appear in the client stream even if the call failed. +CREATE OR REPLACE FUNCTION _debug_log(p_step TEXT, p_detail JSONB) RETURNS void AS $$ +BEGIN + IF current_setting('debug_log.enabled', true) IS DISTINCT FROM 'on' THEN + RETURN; + END IF; + RAISE NOTICE '_DEBUG_LOG: % %', p_step, COALESCE(p_detail::text, 'null'); +END; +$$ LANGUAGE plpgsql; + + -- merge_and_upsert_external_users: detects alias overlaps within temp table -- and against live table, remaps FKs, merges aliases, soft-deletes losers, upserts survivors. -- Returns (loser_id, winner_id) pairs for caller cache eviction. CREATE OR REPLACE FUNCTION merge_and_upsert_external_users(p_temp_table TEXT) RETURNS TABLE(loser_id UUID, winner_id UUID) AS $$ +DECLARE + v_debug BOOLEAN := current_setting('debug_log.enabled', true) = 'on'; + v_row_count BIGINT; + v_edges_total BIGINT; + v_temp_live_count BIGINT; + v_temp_live_no_del_filter BIGINT; + v_temp_live_null_aliases BIGINT; + v_temp_live_id_eq BIGINT; + v_sample JSONB; BEGIN LOCK TABLE config_access, access_reviews, config_access_logs, external_user_groups, external_users, external_groups, external_roles IN SHARE ROW EXCLUSIVE MODE; + IF v_debug THEN + EXECUTE format('SELECT count(*) FROM %I', p_temp_table) INTO v_row_count; + EXECUTE format('SELECT jsonb_agg(to_jsonb(t)) FROM (SELECT * FROM %I LIMIT 5) t', p_temp_table) INTO v_sample; + PERFORM _debug_log('entry', jsonb_build_object( + 'temp_table', p_temp_table, + 'row_count', v_row_count, + 'sample', v_sample + )); + END IF; + + -- Step 0: Normalize the temp table's aliases to match the normalization + -- that the `normalize_aliases` BEFORE INSERT/UPDATE trigger applies to + -- rows stored in external_users (lowercase + sorted + distinct). The + -- edge-build query below uses `tmp.aliases && live.aliases`, which is + -- byte-exact, so any mixed-case alias in the temp table would miss + -- matches against live rows whose aliases the trigger has already + -- normalized. Normalizing up-front keeps the && join symmetric and + -- reliable. + EXECUTE format(' + UPDATE %1$I SET + aliases = ARRAY(SELECT DISTINCT LOWER(elem) FROM unnest(aliases) AS elem ORDER BY LOWER(elem)) + WHERE aliases IS NOT NULL + ', p_temp_table); + + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step0_normalize_temp_aliases', jsonb_build_object('rows_affected', v_row_count)); + END IF; + -- Step 1: Build undirected edge list from alias overlaps EXECUTE format(' CREATE TEMP TABLE _eu_edges ON COMMIT DROP AS @@ -19,7 +81,69 @@ BEGIN ON tmp.aliases && live.aliases AND tmp.id != live.id AND live.deleted_at IS NULL ', p_temp_table); + IF v_debug THEN + -- Post Step 1: edge counts + sample edges. + SELECT count(*) INTO v_edges_total FROM _eu_edges; + PERFORM _debug_log('step1_edges', jsonb_build_object( + 'edge_count', v_edges_total, + 'sample_edges', (SELECT jsonb_agg(jsonb_build_object('id1', id1, 'id2', id2)) + FROM (SELECT * FROM _eu_edges LIMIT 20) e) + )); + + -- Diagnostic re-query: same join with variations to pinpoint which filter + -- is dropping overlaps. Uses unnest + = ANY to bypass the && operator in + -- case &&'s behavior is the culprit. + EXECUTE format(' + SELECT count(*) FROM %1$I tmp JOIN external_users live + ON tmp.aliases && live.aliases AND tmp.id != live.id AND live.deleted_at IS NULL + ', p_temp_table) INTO v_temp_live_count; + EXECUTE format(' + SELECT count(*) FROM %1$I tmp JOIN external_users live + ON tmp.aliases && live.aliases AND tmp.id != live.id + ', p_temp_table) INTO v_temp_live_no_del_filter; + EXECUTE format(' + SELECT count(*) FROM %1$I tmp JOIN external_users live + ON tmp.id = live.id + ', p_temp_table) INTO v_temp_live_id_eq; + EXECUTE format(' + SELECT count(*) FROM %1$I tmp + JOIN LATERAL unnest(COALESCE(tmp.aliases, ARRAY[]::text[])) AS t_alias(val) ON true + JOIN external_users live ON live.deleted_at IS NULL AND live.id <> tmp.id + WHERE t_alias.val = ANY(COALESCE(live.aliases, ARRAY[]::text[])) + ', p_temp_table) INTO v_temp_live_null_aliases; + PERFORM _debug_log('step1_candidates', jsonb_build_object( + 'temp_live_count_using_overlap_op', v_temp_live_count, + 'temp_live_count_without_deleted_filter', v_temp_live_no_del_filter, + 'temp_live_count_with_id_eq', v_temp_live_id_eq, + 'temp_live_count_via_unnest_any', v_temp_live_null_aliases + )); + END IF; + IF NOT EXISTS (SELECT 1 FROM _eu_edges) THEN + IF v_debug THEN + PERFORM _debug_log('short_circuit_entry', jsonb_build_object('reason', 'empty _eu_edges')); + + -- Post-hoc: find any temp↔live pair that overlaps via unnest + = ANY. + -- If this returns rows, the && operator missed them and the short- + -- circuit branch will fail with a partial unique index violation. + EXECUTE format(' + SELECT jsonb_agg(row_to_json(r)) + FROM ( + SELECT tmp.id AS tmp_id, tmp.aliases AS tmp_aliases, + live.id AS live_id, live.aliases AS live_aliases, + t_alias.val AS shared_alias + FROM %1$I tmp + JOIN LATERAL unnest(COALESCE(tmp.aliases, ARRAY[]::text[])) AS t_alias(val) ON true + JOIN external_users live + ON live.deleted_at IS NULL + AND live.id <> tmp.id + AND t_alias.val = ANY(COALESCE(live.aliases, ARRAY[]::text[])) + LIMIT 20 + ) r + ', p_temp_table) INTO v_sample; + PERFORM _debug_log('short_circuit_collisions_via_unnest', v_sample); + END IF; + EXECUTE format(' INSERT INTO external_users (id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by) SELECT id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by FROM %I @@ -60,6 +184,33 @@ BEGIN INSERT INTO _eu_merges (loser_id, winner_id) SELECT node, leader FROM _eu_comp WHERE node != leader; + IF v_debug THEN + PERFORM _debug_log('step3_merges', jsonb_build_object( + 'comp_count', (SELECT count(*) FROM _eu_comp), + 'merges_count', (SELECT count(*) FROM _eu_merges), + 'sample_merges', (SELECT jsonb_agg(jsonb_build_object('loser', m.loser_id, 'winner', m.winner_id)) + FROM (SELECT * FROM _eu_merges LIMIT 20) m) + )); + END IF; + + -- Step 3a: Pre-soft-delete any live losers BEFORE Step 3b inserts the temp + -- winners. Without this, the partial unique index on aliases still contains + -- the live loser row at the moment the temp winner is inserted, and the + -- INSERT fires `external_users_aliases_key` whenever the two rows share an + -- alias set (common: one was produced from the same scraper source with a + -- different hash, and the rows carry the same email/descriptor aliases). + -- The losers are moved out of the `WHERE deleted_at IS NULL` partial index + -- here; their alias union is still merged into the winners later in Step 5. + UPDATE external_users SET deleted_at = NOW() + FROM _eu_merges mp + WHERE external_users.id = mp.loser_id + AND external_users.deleted_at IS NULL; + + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step3a_presoft_delete_live_losers', jsonb_build_object('rows_affected', v_row_count)); + END IF; + -- Step 3b: Pre-insert winners from temp table so FK remaps don't violate constraints EXECUTE format(' INSERT INTO external_users (id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by) @@ -68,6 +219,11 @@ BEGIN ON CONFLICT (id) DO NOTHING ', p_temp_table); + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step3b_preinsert', jsonb_build_object('rows_affected', v_row_count)); + END IF; + -- Step 4: Remap FKs in bulk without violating unique constraints CREATE TEMP TABLE _eu_ca_dups (id TEXT PRIMARY KEY) ON COMMIT DROP; INSERT INTO _eu_ca_dups (id) @@ -141,22 +297,40 @@ BEGIN DELETE FROM external_user_groups USING _eu_merges mp WHERE external_user_groups.external_user_id = mp.loser_id; - -- Step 5: Merge aliases from losers into winners in live table - UPDATE external_users SET - aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_users.aliases || agg.all_aliases) ORDER BY 1), '{}'::text[]) - FROM ( - SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases - FROM _eu_merges mp JOIN external_users eu ON eu.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(eu.aliases, '{}'::text[])) AS a - GROUP BY mp.winner_id - ) agg - WHERE external_users.id = agg.winner_id; + -- Step 5: Merge aliases from losers into winners in the live table. + -- We also append `loser.id::text` to the winner's alias union so that + -- future lookups by the loser's old id can recover the winner. + -- entity.aliases never contains entity.id for live rows; loser ids only + -- become aliases here, at the moment they stop being canonical. + -- + -- A loser may live in the temp table (LEFT JOIN tmp), the live table + -- (LEFT JOIN external_users) or both — we union all available alias + -- sources together with the loser id itself. + EXECUTE format(' + UPDATE external_users SET + aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_users.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) + FROM ( + SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases + FROM _eu_merges mp + LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id + LEFT JOIN external_users live_src ON live_src.id = mp.loser_id + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a + GROUP BY mp.winner_id + ) agg + WHERE external_users.id = agg.winner_id + ', p_temp_table); -- Step 6: Soft-delete losers UPDATE external_users SET deleted_at = NOW() FROM _eu_merges mp WHERE external_users.id = mp.loser_id; - -- Step 7: Consolidate temp table (merge aliases from all losers - both temp and live - into temp winners) + -- Step 7: Consolidate temp table (merge aliases from all losers - both + -- temp and live - into temp winners). As in Step 5, also append the + -- loser id itself so it remains lookupable via aliases after the merge. EXECUTE format(' UPDATE %1$I t SET aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(t.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) @@ -165,13 +339,22 @@ BEGIN FROM _eu_merges mp LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id LEFT JOIN external_users live_src ON live_src.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(tmp_src.aliases, ''{}''::text[]) || COALESCE(live_src.aliases, ''{}''::text[])) AS a + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a GROUP BY mp.winner_id ) agg WHERE t.id = agg.winner_id ', p_temp_table); EXECUTE format('DELETE FROM %I USING _eu_merges mp WHERE id = mp.loser_id', p_temp_table); + IF v_debug THEN + EXECUTE format('SELECT count(*) FROM %I', p_temp_table) INTO v_row_count; + PERFORM _debug_log('step7_post_delete_temp', jsonb_build_object('remaining_temp_rows', v_row_count)); + END IF; + -- Step 8: Upsert survivors EXECUTE format(' INSERT INTO external_users (id, aliases, name, account_id, user_type, email, scraper_id, created_at, updated_at, created_by) @@ -183,6 +366,11 @@ BEGIN updated_at = EXCLUDED.updated_at, deleted_at = NULL ', p_temp_table); + IF v_debug THEN + GET DIAGNOSTICS v_row_count = ROW_COUNT; + PERFORM _debug_log('step8_upsert', jsonb_build_object('rows_affected', v_row_count)); + END IF; + RETURN QUERY SELECT mp.loser_id, mp.winner_id FROM _eu_merges mp; END; $$ LANGUAGE plpgsql; @@ -194,6 +382,17 @@ BEGIN LOCK TABLE config_access, access_reviews, config_access_logs, external_user_groups, external_users, external_groups, external_roles IN SHARE ROW EXCLUSIVE MODE; + -- Step 0: Normalize temp aliases to match the normalize_aliases() trigger + -- on external_groups (lowercase + sorted + distinct). The edge-build below + -- uses byte-exact && which would otherwise miss any temp row whose aliases + -- differ from the normalized live row only by case. See the equivalent + -- Step 0 in merge_and_upsert_external_users for details. + EXECUTE format(' + UPDATE %1$I SET + aliases = ARRAY(SELECT DISTINCT LOWER(elem) FROM unnest(aliases) AS elem ORDER BY LOWER(elem)) + WHERE aliases IS NOT NULL + ', p_temp_table); + EXECUTE format(' CREATE TEMP TABLE _eg_edges ON COMMIT DROP AS SELECT DISTINCT a.id AS id1, b.id AS id2 @@ -240,6 +439,15 @@ BEGIN CREATE TEMP TABLE _eg_merges (loser_id UUID PRIMARY KEY, winner_id UUID) ON COMMIT DROP; INSERT INTO _eg_merges SELECT node, leader FROM _eg_comp WHERE node != leader; + -- Step 3a: Pre-soft-delete live losers BEFORE pre-insert so the partial + -- unique index doesn't fire on a winner with identical aliases to a + -- still-active loser. See the equivalent Step 3a in + -- merge_and_upsert_external_users for details. + UPDATE external_groups SET deleted_at = NOW() + FROM _eg_merges mp + WHERE external_groups.id = mp.loser_id + AND external_groups.deleted_at IS NULL; + EXECUTE format(' INSERT INTO external_groups (id, aliases, name, account_id, scraper_id, group_type, created_at, updated_at) SELECT id, aliases, name, account_id, scraper_id, group_type, created_at, updated_at @@ -292,14 +500,25 @@ BEGIN DELETE FROM external_user_groups USING _eg_merges mp WHERE external_user_groups.external_group_id = mp.loser_id; - UPDATE external_groups SET - aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_groups.aliases || agg.all_aliases) ORDER BY 1), '{}'::text[]) - FROM ( - SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases - FROM _eg_merges mp JOIN external_groups eg ON eg.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(eg.aliases, '{}'::text[])) AS a - GROUP BY mp.winner_id - ) agg WHERE external_groups.id = agg.winner_id; + -- Merge loser aliases (and the loser id itself) into the winner. See the + -- equivalent step in merge_and_upsert_external_users for the reasoning, + -- including why we LEFT JOIN both temp and live as alias sources. + EXECUTE format(' + UPDATE external_groups SET + aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_groups.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) + FROM ( + SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases + FROM _eg_merges mp + LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id + LEFT JOIN external_groups live_src ON live_src.id = mp.loser_id + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a + GROUP BY mp.winner_id + ) agg WHERE external_groups.id = agg.winner_id + ', p_temp_table); UPDATE external_groups SET deleted_at = NOW() FROM _eg_merges mp WHERE external_groups.id = mp.loser_id; @@ -312,7 +531,11 @@ BEGIN FROM _eg_merges mp LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id LEFT JOIN external_groups live_src ON live_src.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(tmp_src.aliases, ''{}''::text[]) || COALESCE(live_src.aliases, ''{}''::text[])) AS a + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a GROUP BY mp.winner_id ) agg WHERE t.id = agg.winner_id ', p_temp_table); @@ -339,6 +562,14 @@ BEGIN LOCK TABLE config_access, access_reviews, config_access_logs, external_user_groups, external_users, external_groups, external_roles IN SHARE ROW EXCLUSIVE MODE; + -- Step 0: Normalize temp aliases to match the normalize_aliases() trigger + -- on external_roles. See Step 0 in merge_and_upsert_external_users. + EXECUTE format(' + UPDATE %1$I SET + aliases = ARRAY(SELECT DISTINCT LOWER(elem) FROM unnest(aliases) AS elem ORDER BY LOWER(elem)) + WHERE aliases IS NOT NULL + ', p_temp_table); + EXECUTE format(' CREATE TEMP TABLE _er_edges ON COMMIT DROP AS SELECT DISTINCT a.id AS id1, b.id AS id2 @@ -386,6 +617,13 @@ BEGIN CREATE TEMP TABLE _er_merges (loser_id UUID PRIMARY KEY, winner_id UUID) ON COMMIT DROP; INSERT INTO _er_merges SELECT node, leader FROM _er_comp WHERE node != leader; + -- Step 3a: Pre-soft-delete live losers BEFORE pre-insert. See Step 3a in + -- merge_and_upsert_external_users. + UPDATE external_roles SET deleted_at = NOW() + FROM _er_merges mp + WHERE external_roles.id = mp.loser_id + AND external_roles.deleted_at IS NULL; + EXECUTE format(' INSERT INTO external_roles (id, aliases, name, account_id, role_type, description, scraper_id, application_id, created_at, updated_at) SELECT id, aliases, name, account_id, role_type, description, scraper_id, application_id, created_at, updated_at @@ -432,14 +670,25 @@ BEGIN UPDATE access_reviews SET external_role_id = mp.winner_id FROM _er_merges mp WHERE access_reviews.external_role_id = mp.loser_id; - UPDATE external_roles SET - aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_roles.aliases || agg.all_aliases) ORDER BY 1), '{}'::text[]) - FROM ( - SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases - FROM _er_merges mp JOIN external_roles er ON er.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(er.aliases, '{}'::text[])) AS a - GROUP BY mp.winner_id - ) agg WHERE external_roles.id = agg.winner_id; + -- Merge loser aliases (and the loser id itself) into the winner. See the + -- equivalent step in merge_and_upsert_external_users for the reasoning, + -- including why we LEFT JOIN both temp and live as alias sources. + EXECUTE format(' + UPDATE external_roles SET + aliases = NULLIF(ARRAY(SELECT DISTINCT unnest FROM unnest(external_roles.aliases || agg.all_aliases) ORDER BY 1), ''{}''::text[]) + FROM ( + SELECT mp.winner_id, array_agg(DISTINCT a) AS all_aliases + FROM _er_merges mp + LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id + LEFT JOIN external_roles live_src ON live_src.id = mp.loser_id + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a + GROUP BY mp.winner_id + ) agg WHERE external_roles.id = agg.winner_id + ', p_temp_table); UPDATE external_roles SET deleted_at = NOW() FROM _er_merges mp WHERE external_roles.id = mp.loser_id; @@ -452,7 +701,11 @@ BEGIN FROM _er_merges mp LEFT JOIN %1$I tmp_src ON tmp_src.id = mp.loser_id LEFT JOIN external_roles live_src ON live_src.id = mp.loser_id - CROSS JOIN LATERAL unnest(COALESCE(tmp_src.aliases, ''{}''::text[]) || COALESCE(live_src.aliases, ''{}''::text[])) AS a + CROSS JOIN LATERAL unnest( + COALESCE(tmp_src.aliases, ''{}''::text[]) + || COALESCE(live_src.aliases, ''{}''::text[]) + || ARRAY[mp.loser_id::text] + ) AS a GROUP BY mp.winner_id ) agg WHERE t.id = agg.winner_id ', p_temp_table); diff --git a/views/047_change_groups.sql b/views/047_change_groups.sql new file mode 100644 index 000000000..dabec2dd2 --- /dev/null +++ b/views/047_change_groups.sql @@ -0,0 +1,69 @@ +-- dependsOn: functions/drop.sql, views/030_config_changes.sql + +-- Maintains change_groups.member_count, last_member_at, ended_at and started_at +-- when a config_changes row is attached to a group via group_id. +-- Runs on INSERT and on UPDATE OF group_id only, so the dedup UPDATE path in +-- config_changes_update_trigger() does not re-evaluate membership. +CREATE OR REPLACE FUNCTION change_groups_maintain_members() +RETURNS TRIGGER AS $$ +BEGIN + IF NEW.group_id IS NULL THEN + RETURN NEW; + END IF; + + IF TG_OP = 'UPDATE' AND OLD.group_id IS NOT DISTINCT FROM NEW.group_id THEN + RETURN NEW; + END IF; + + UPDATE change_groups g + SET + member_count = g.member_count + 1, + last_member_at = GREATEST(g.last_member_at, NEW.created_at), + ended_at = CASE + WHEN g.ended_at IS NULL THEN NULL + ELSE GREATEST(g.ended_at, NEW.created_at) + END, + started_at = CASE + WHEN g.member_count = 0 THEN NEW.created_at + ELSE LEAST(g.started_at, NEW.created_at) + END, + updated_at = now() + WHERE g.id = NEW.group_id; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS trg_change_groups_maintain_members_ins ON config_changes; +CREATE TRIGGER trg_change_groups_maintain_members_ins +AFTER INSERT ON config_changes +FOR EACH ROW +EXECUTE FUNCTION change_groups_maintain_members(); + +DROP TRIGGER IF EXISTS trg_change_groups_maintain_members_upd ON config_changes; +CREATE TRIGGER trg_change_groups_maintain_members_upd +AFTER UPDATE OF group_id ON config_changes +FOR EACH ROW +EXECUTE FUNCTION change_groups_maintain_members(); + +-- Aggregated view used by the query layer / UI. +CREATE OR REPLACE VIEW change_groups_summary AS +SELECT + g.id, + g.type, + g.summary, + g.source, + g.rule_name, + g.status, + g.started_at, + g.ended_at, + g.last_member_at, + g.member_count, + COUNT(DISTINCT cc.config_id) AS distinct_config_count, + EXTRACT(EPOCH FROM (COALESCE(g.ended_at, g.last_member_at) - g.started_at)) AS duration_seconds, + g.details, + g.created_at, + g.updated_at +FROM change_groups g +LEFT JOIN config_changes cc ON cc.group_id = g.id +GROUP BY g.id;