Skip to content

feat: scope config property updates by creator #2171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
adityathebe wants to merge 2 commits into from
Closed

feat: scope config property updates by creator #2171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d3bef31
feat(config): scope property updates by creator
adityathebe May 5, 2026
82d8b7e
update benchmark
adityathebe Apr 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions bench/config_items_save_results_bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
v1 "github.com/flanksource/config-db/api/v1"
"github.com/flanksource/config-db/db"
dutyModels "github.com/flanksource/duty/models"
"github.com/flanksource/duty/types"
"github.com/google/uuid"
)

Expand Down Expand Up @@ -202,12 +203,29 @@ func buildScrapeResults(size int, dataset string, revision int) []v1.ScrapeResul
"port": 8080,
},
},
Properties: buildScrapeResultProperties(revision),
})
}

return results
}

func buildScrapeResultProperties(revision int) types.Properties {
payload := strings.Repeat("x", 16*1024)
properties := make(types.Properties, 0, 6)
for i := range 6 {
properties = append(properties, &types.Property{
Name: fmt.Sprintf("bench-property-%d", i),
Label: fmt.Sprintf("Bench Property %d", i),
Type: "text",
Text: fmt.Sprintf("%s-%d-%d", payload, revision, i),
Tooltip: fmt.Sprintf("bench property tooltip %d", i),
Order: i,
})
}
return properties
}

func cleanupBenchRows(tb testing.TB, scraperID uuid.UUID) {
tb.Helper()

Expand Down
10 changes: 9 additions & 1 deletion db/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,6 @@ func NewConfigItemFromResult(ctx api.ScrapeContext, result v1.ScrapeResult) (*mo
Name: &result.Name,
Source: &result.Source,
Labels: (*types.JSONStringMap)(&result.Labels),
Properties: &result.Properties,
Config: &dataStr,
Ready: result.Ready,
Parents: result.Parents,
Expand All @@ -160,6 +159,15 @@ func NewConfigItemFromResult(ctx api.ScrapeContext, result v1.ScrapeResult) (*mo
ci.ScraperID = nil
}

// Scraper-owned config results always express the scraper's current property slice.
// A nil/empty result.Properties therefore means "the scraper now owns zero properties"
// and must remove stale scraper-created properties while preserving user/other-scraper properties.
if ci.ScraperID != nil && *ci.ScraperID != uuid.Nil {
ci.Properties = lo.ToPtr(dutyModels.NewOwnedProperties(result.Properties))
} else if result.Properties != nil {
ci.Properties = lo.ToPtr(dutyModels.NewOwnedProperties(result.Properties))
}

ci.Tags = types.JSONStringMap(result.Tags)
// If the config result hasn't specified an id for the config,
// we try to use the external id as the primary key of the config item.
Expand Down
50 changes: 25 additions & 25 deletions db/models/config_item.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,31 +17,31 @@ import (
// ConfigItem represents the config item database table
// Deprecated: Use models.ConfigItem from duty.
type ConfigItem struct {
ID string `gorm:"primaryKey;unique_index;not null;column:id;default:generate_ulid()" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type string `gorm:"column:type" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Ready bool `json:"ready,omitempty" `
Health *models.Health `json:"health,omitempty"`
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Labels *types.JSONStringMap `gorm:"column:labels;default:null" json:"labels,omitempty"`
Tags types.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty"`
Properties *types.Properties `gorm:"column:properties;default:null" json:"properties,omitempty"`
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt *time.Time `gorm:"column:updated_at;autoUpdateTime:false;<-:update" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
DeleteReason v1.ConfigDeleteReason `gorm:"column:delete_reason" json:"delete_reason"`
ID string `gorm:"primaryKey;unique_index;not null;column:id;default:generate_ulid()" json:"id" `
ScraperID *uuid.UUID `gorm:"column:scraper_id;default:null" json:"scraper_id,omitempty"`
ConfigClass string `gorm:"column:config_class;default:''" json:"config_class" `
ExternalID pq.StringArray `gorm:"column:external_id;type:[]text" json:"external_id,omitempty" `
Type string `gorm:"column:type" json:"type,omitempty" `
Status *string `gorm:"column:status;default:null" json:"status,omitempty" `
Ready bool `json:"ready,omitempty" `
Health *models.Health `json:"health,omitempty"`
Name *string `gorm:"column:name;default:null" json:"name,omitempty" `
Description *string `gorm:"column:description;default:null" json:"description,omitempty" `
Config *string `gorm:"column:config;default:null" json:"config,omitempty" `
Source *string `gorm:"column:source;default:null" json:"source,omitempty" `
ParentID *string `gorm:"column:parent_id;default:null" json:"parent_id,omitempty"`
Path string `gorm:"column:path;default:null" json:"path,omitempty"`
CostPerMinute float64 `gorm:"column:cost_per_minute;default:null" json:"cost_per_minute,omitempty"`
CostTotal1d float64 `gorm:"column:cost_total_1d;default:null" json:"cost_total_1d,omitempty"`
CostTotal7d float64 `gorm:"column:cost_total_7d;default:null" json:"cost_total_7d,omitempty"`
CostTotal30d float64 `gorm:"column:cost_total_30d;default:null" json:"cost_total_30d,omitempty"`
Labels *types.JSONStringMap `gorm:"column:labels;default:null" json:"labels,omitempty"`
Tags types.JSONStringMap `gorm:"column:tags;default:null" json:"tags,omitempty"`
Properties *models.OwnedProperties `gorm:"column:properties;default:null" json:"properties,omitempty"`
CreatedAt time.Time `gorm:"column:created_at" json:"created_at"`
UpdatedAt *time.Time `gorm:"column:updated_at;autoUpdateTime:false;<-:update" json:"updated_at"`
DeletedAt *time.Time `gorm:"column:deleted_at" json:"deleted_at"`
DeleteReason v1.ConfigDeleteReason `gorm:"column:delete_reason" json:"delete_reason"`

Parents []v1.ConfigExternalKey `gorm:"-" json:"parents,omitempty"`
Children []v1.ConfigExternalKey `gorm:"-" json:"children,omitempty"`
Expand Down
102 changes: 77 additions & 25 deletions db/update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,18 +109,6 @@ func mapStringEqual(a, b map[string]string) bool {
return true
}

func mapEqual(a, b map[string]any) bool {
if len(a) != len(b) {
return false
}
for k, v := range a {
if b[k] != v {
return false
}
}
return true
}

func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result *v1.ScrapeResult, ci, existing *models.ConfigItem) (bool, []*models.ConfigChange, error) {
ci.ID = existing.ID
updates := make(map[string]any)
Expand Down Expand Up @@ -221,16 +209,51 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result *v1.Scrap
summary.AddWarning(ci.Type, fmt.Sprintf("updated scraper_id of config[%s] from %s to %s", ci, existing.ScraperID, ci.ScraperID))
}

if ci.Properties != nil && len(*ci.Properties) > 0 && (existing.Properties == nil || !mapEqual(ci.Properties.AsMap(), existing.Properties.AsMap())) {
updates["properties"] = *ci.Properties
}
// nil properties mean "not observed / no opinion".
// empty properties mean "observed and now owns zero properties", so remove this creator's slice.
propertyUpdateNeeded := ci.Properties != nil

if len(updates) == 0 {
if len(updates) == 0 && !propertyUpdateNeeded {
return false, changes, nil
}

if err := ctx.DutyContext().DB().Model(ci).Updates(updates).Error; err != nil {
return false, nil, errors.Wrapf(dutydb.ErrorDetails(err), "unable to update config item: %s", ci)
fieldsChanged := len(updates) > 0
propertyChanged := false
if err := ctx.DutyContext().DB().Transaction(func(tx *gorm.DB) error {
if len(updates) > 0 {
if err := tx.Model(ci).Updates(updates).Error; err != nil {
return dutydb.ErrorDetails(err)
}
}

if propertyUpdateNeeded {
createdBy := ctx.ScraperID()
if ci.ScraperID != nil && *ci.ScraperID != uuid.Nil {
createdBy = ci.ScraperID.String()
}
if createdBy == "" {
return nil
}

configID, err := uuid.Parse(ci.ID)
if err != nil {
return fmt.Errorf("invalid config id %q: %w", ci.ID, err)
}
createdByID, err := uuid.Parse(createdBy)
if err != nil {
return fmt.Errorf("invalid property creator id %q: %w", createdBy, err)
}

result, err := dutyModels.UpdateConfigItemProperties(tx, configID, dutyModels.PropertyCreatorTypeScraper, createdByID, ci.Properties.AsProperties())
if err != nil {
return dutydb.ErrorDetails(err)
}
ci.Properties = &result.Properties
propertyChanged = result.Changed
}
return nil
}); err != nil {
return false, nil, errors.Wrapf(err, "unable to update config item: %s", ci)
}

if isDeleted {
Expand All @@ -241,7 +264,7 @@ func updateCI(ctx api.ScrapeContext, summary *v1.ScrapeSummary, result *v1.Scrap
).Add(1)
}

return true, changes, nil
return fieldsChanged || propertyChanged, changes, nil
}

func shouldExcludeChange(ctx api.ScrapeContext, result *v1.ScrapeResult, changeResult v1.ChangeResult) (bool, error) {
Expand Down Expand Up @@ -576,13 +599,42 @@ func saveResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (v1.ScrapeSum
summary.AddChangeSummary(configType, cs)
}

// NOTE: On duplicate primary key do nothing
// because an incremental scraper might have already inserted the config item.
if err := ctx.DB().
Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "id"}}, DoNothing: true}).
CreateInBatches(extractResult.newConfigs, configItemsBulkInsertSize).Error; err != nil {
return summary, ctx.Oops().Wrapf(dutydb.ErrorDetails(err), "failed to create config items")
newConfigProperties := map[string]dutyModels.OwnedProperties{}
for _, config := range extractResult.newConfigs {
if config.Properties != nil && scraperID != nil && *scraperID != uuid.Nil {
newConfigProperties[config.ID] = *config.Properties
config.Properties = nil
}
}

if err := ctx.DB().Transaction(func(tx *gorm.DB) error {
// NOTE: On duplicate primary key do nothing
// because an incremental scraper might have already inserted the config item.
if err := tx.
Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "id"}}, DoNothing: true}).
CreateInBatches(extractResult.newConfigs, configItemsBulkInsertSize).Error; err != nil {
return dutydb.ErrorDetails(err)
}

for _, config := range extractResult.newConfigs {
if props, ok := newConfigProperties[config.ID]; ok {
configID, err := uuid.Parse(config.ID)
if err != nil {
return fmt.Errorf("invalid config id %q: %w", config.ID, err)
}

result, err := dutyModels.UpdateConfigItemProperties(tx, configID, dutyModels.PropertyCreatorTypeScraper, *scraperID, props.AsProperties())
if err != nil {
return dutydb.ErrorDetails(err)
}
config.Properties = &result.Properties
}
}
return nil
}); err != nil {
return summary, ctx.Oops().Wrapf(err, "failed to create config items")
}

for _, config := range extractResult.newConfigs {
summary.AddInserted(config.Type)
ctx.TempCache().Insert(*config)
Expand Down
14 changes: 3 additions & 11 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ require (
github.com/flanksource/clicky v1.21.8
github.com/flanksource/commons v1.51.4
github.com/flanksource/deps v1.0.28
github.com/flanksource/duty v1.0.1301
github.com/flanksource/duty v1.0.1302-0.20260505080647-4f6025d97824
github.com/flanksource/gomplate/v3 v3.24.79
github.com/flanksource/is-healthy v1.0.87
github.com/flanksource/ketall v1.1.9
Expand Down Expand Up @@ -163,12 +163,10 @@ require (
github.com/aws/aws-sdk-go-v2/service/kms v1.49.5 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.10 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/bmatcuk/doublestar v1.3.4 // indirect
github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
github.com/buger/jsonparser v1.1.2 // indirect
github.com/casbin/casbin/v2 v2.135.0 // indirect
github.com/casbin/casbin/v3 v3.8.1 // indirect
github.com/casbin/gorm-adapter/v3 v3.41.0 // indirect
Expand Down Expand Up @@ -270,7 +268,6 @@ require (
github.com/hashicorp/hcl/v2 v2.24.0 // indirect
github.com/henvic/httpretty v0.1.4 // indirect
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
github.com/invopop/jsonschema v0.13.0 // indirect
github.com/itchyny/gojq v0.12.19 // indirect
github.com/itchyny/timefmt-go v0.1.8 // indirect
github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect
Expand All @@ -295,7 +292,6 @@ require (
github.com/lrita/cmap v0.0.0-20231108122212-cb084a67f554 // indirect
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
github.com/mailru/easyjson v0.9.1 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/microsoft/kiota-abstractions-go v1.9.4 // indirect
github.com/microsoft/kiota-authentication-azure-go v1.3.1 // indirect
Expand Down Expand Up @@ -360,12 +356,8 @@ require (
github.com/vadimi/go-http-ntlm v1.0.3 // indirect
github.com/vadimi/go-http-ntlm/v2 v2.5.0 // indirect
github.com/vadimi/go-ntlm v1.2.1 // indirect
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/xuri/efp v0.0.1 // indirect
Expand Down Expand Up @@ -523,11 +515,11 @@ require (
// replace github.com/flanksource/clicky => ../clicky
// replace github.com/flanksource/commons => ../commons
// replace github.com/flanksource/deps => ../deps
// replace github.com/flanksource/duty => ../duty

// replace github.com/flanksource/gomplate/v3 => ../gomplate
// replace github.com/flanksource/ketall => ../ketall
// replace github.com/flanksource/postq => ../postq
// replace github.com/flanksource/is-healthy => ../is-healthy

// replace github.com/flanksource/duty => ../duty

replace github.com/glebarez/sqlite => github.com/clarkmcc/gorm-sqlite v0.0.0-20240426202654-00ed082c0311
Loading
Loading